{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":6799,"databundleVersionId":4225553,"sourceType":"competition"},{"sourceId":7262345,"sourceType":"datasetVersion","datasetId":4209040}],"dockerImageVersionId":30626,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade keras-cv\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers,utils\nimport math\nimport numpy as np\nfrom matplotlib import pyplot as plt\nimport matplotlib.image as mpimg\nimport os     \nimport pickle\nimport shutil  \nfrom IPython.display import clear_output\nfrom keras_cv.layers import TransformerEncoder\nfrom keras_cv.layers import PatchingAndEmbedding","metadata":{"execution":{"iopub.status.busy":"2024-02-14T04:42:16.009411Z","iopub.execute_input":"2024-02-14T04:42:16.009734Z","iopub.status.idle":"2024-02-14T04:42:16.013746Z","shell.execute_reply.started":"2024-02-14T04:42:16.009706Z","shell.execute_reply":"2024-02-14T04:42:16.012949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# detect and init the TPU\nresolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='local')\ntf.config.experimental_connect_to_cluster(resolver)\ntf.tpu.experimental.initialize_tpu_system(resolver)\n\n\n# instantiate a distribution strategy\ntpu_strategy = tf.distribute.TPUStrategy(resolver)\n\nprint(\"Tensorflow version \" + tf.__version__)\n\n\nprint(\"REPLICAS: \", tpu_strategy.num_replicas_in_sync)\nprint('DEVICES AVAILABLE: {}'.format(tpu_strategy.num_replicas_in_sync))\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T04:37:04.435289Z","iopub.execute_input":"2024-02-14T04:37:04.435818Z","iopub.status.idle":"2024-02-14T04:37:13.312905Z","shell.execute_reply.started":"2024-02-14T04:37:04.435789Z","shell.execute_reply":"2024-02-14T04:37:13.312143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@keras.saving.register_keras_serializable()\ndef parse_weights(weights, include_top, model_type):\n    if not weights:\n        return weights\n    if weights.startswith(\"gs://\"):\n        weights = weights.replace(\"gs://\", \"https://storage.googleapis.com/\")\n        return utils.get_file(\n            origin=weights,\n            cache_subdir=\"models\",\n        )\n    if tf.io.gfile.exists(weights):\n        return weights\n    if weights in ALIASES[model_type]:\n        weights = ALIASES[model_type][weights]\n    if weights in WEIGHTS_CONFIG[model_type]:\n        if not include_top:\n            weights = weights + \"-notop\"\n        return utils.get_file(\n            origin=f\"{BASE_PATH}/{model_type}/{weights}.h5\",\n            cache_subdir=\"models\",\n            file_hash=WEIGHTS_CONFIG[model_type][weights],\n        )\n\n    raise ValueError(\n        \"The `weights` argument should be either `None`, a the path to the \"\n        \"weights file to be loaded, or the name of pre-trained weights from \"\n        \"https://github.com/keras-team/keras-cv/blob/master/keras_cv/models/weights.py. \"  # noqa: E501\n        f\"Invalid `weights` argument: {weights}\"\n    )\n\n\nBASE_PATH = \"https://storage.googleapis.com/keras-cv/models\"\n\nALIASES = {\n    \"convmixer_512_16\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"cspdarknetl\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"cspdarknettiny\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"darknet53\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"deeplabv3\": {\n        \"voc\": \"voc/segmentation-v0\",\n    },\n    \"densenet121\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"densenet169\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"densenet201\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"resnet50\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"resnet50v2\": {\n        \"imagenet\": \"imagenet/classification-v2\",\n        \"imagenet/classification\": \"imagenet/classification-v2\",\n    },\n    \"vittiny16\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"vits16\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"vitb16\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"vitl16\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"vits32\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n    \"vitb32\": {\n        \"imagenet\": \"imagenet/classification-v0\",\n        \"imagenet/classification\": \"imagenet/classification-v0\",\n    },\n}\n\nWEIGHTS_CONFIG = {\n    \"convmixer_512_16\": {\n        \"imagenet/classification-v0\": \"861f3080dc383f7936d3df89691aadea05eee6acaa4a0b60aa70dd657df915ee\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"aa08c7fa9ca6ec045c4783e1248198dbe1bc141e2ae788e712de471c0370822c\",  # noqa: E501\n    },\n    \"cspdarknetl\": {\n        \"imagenet/classification-v0\": \"8bdc3359222f0d26f77aa42c4e97d67a05a1431fe6c448ceeab9a9c5a34ff804\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"9303aabfadffbff8447171fce1e941f96d230d8f3cef30d3f05a9c85097f8f1e\",  # noqa: E501\n    },\n    \"cspdarknettiny\": {\n        \"imagenet/classification-v0\": \"c17fe6d7b597f2eb25e42fbd97ec58fb1dad753ba18920cc27820953b7947704\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"0007ae82c95be4d4aef06368a7c38e006381324d77e5df029b04890e18a8ad19\",  # noqa: E501\n    },\n    \"darknet53\": {\n        \"imagenet/classification-v0\": \"7bc5589f7f7f7ee3878e61ab9323a71682bfb617eb57f530ca8757c742f00c77\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"8dcce43163e4b4a63e74330ba1902e520211db72d895b0b090b6bfe103e7a8a5\",  # noqa: E501\n    },\n    \"deeplabv3\": {\n        \"voc/segmentation-v0\": \"732042e8b6c9ddba3d51c861f26dc41865187e9f85a0e5d43dfef75a405cca18\",  # noqa: E501\n    },\n    \"densenet121\": {\n        \"imagenet/classification-v0\": \"13de3d077ad9d9816b9a0acc78215201d9b6e216c7ed8e71d69cc914f8f0775b\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"709afe0321d9f2b2562e562ff9d0dc44cca10ed09e0e2cfba08d783ff4dab6bf\",  # noqa: E501\n    },\n    \"densenet169\": {\n        \"imagenet/classification-v0\": \"4cd2a661d0cb2378574073b23129ee4d06ea53c895c62a8863c44ee039e236a1\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"a99d1bb2cbe1a59a1cdd1f435fb265453a97c2a7b723d26f4ebee96e5fb49d62\",  # noqa: E501\n    },\n    \"densenet201\": {\n        \"imagenet/classification-v0\": \"3b6032e744e5e5babf7457abceaaba11fcd449fe2d07016ae5076ac3c3c6cf0c\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"c1189a934f12c1a676a9cf52238e5994401af925e2adfc0365bad8133c052060\",  # noqa: E501\n    },\n    \"resnet50\": {\n        \"imagenet/classification-v0\": \"1525dc1ce580239839ba6848c0f1b674dc89cb9ed73c4ed49eba355b35eac3ce\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"dc5f6d8f929c78d0fc192afecc67b11ac2166e9d8b9ef945742368ae254c07af\",  # noqa: E501\n    },\n    \"resnet50v2\": {\n        \"imagenet/classification-v0\": \"11bde945b54d1dca65101be2648048abca8a96a51a42820d87403486389790db\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"5b4aca4932c433d84f6aef58135472a4312ed2fa565d53fedcd6b0c24b54ab4a\",  # noqa: E501\n        \"imagenet/classification-v1\": \"a32e5d9998e061527f6f947f36d8e794ad54dad71edcd8921cda7804912f3ee7\",  # noqa: E501\n        \"imagenet/classification-v1-notop\": \"ac46b82c11070ab2f69673c41fbe5039c9eb686cca4f34cd1d79412fd136f1ae\",  # noqa: E501\n        \"imagenet/classification-v2\": \"5ee5a8ac650aaa59342bc48ffe770e6797a5550bcc35961e1d06685292c15921\",  # noqa: E501\n        \"imagenet/classification-v2-notop\": \"e711c83d6db7034871f6d345a476c8184eab99dbf3ffcec0c1d8445684890ad9\",  # noqa: E501\n    },\n    \"vittiny16\": {\n        \"imagenet/classification-v0\": \"c8227fde16ec8c2e7ab886169b11b4f0ca9af2696df6d16767db20acc9f6e0dd\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"aa4d727e3c6bd30b20f49d3fa294fb4bbef97365c7dcb5cee9c527e4e83c8f5b\",  # noqa: E501\n    },\n    \"vits16\": {\n        \"imagenet/classification-v0\": \"4a66a1a70a879ff33a3ca6ca30633b9eadafea84b421c92174557eee83e088b5\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"8d0111eda6692096676a5453abfec5d04c79e2de184b04627b295f10b1949745\",  # noqa: E501\n    },\n    \"vitb16\": {\n        \"imagenet/classification-v0\": \"6ab4e08c773e08de42023d963a97e905ccba710e2c05ef60c0971978d4a8c41b\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"4a1bdd32889298471cb4f30882632e5744fd519bf1a1525b1fa312fe4ea775ed\",  # noqa: E501\n    },\n    \"vitl16\": {\n        \"imagenet/classification-v0\": \"5a98000f848f2e813ea896b2528983d8d956f8c4b76ceed0b656219d5b34f7fb\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"40d237c44f14d20337266fce6192c00c2f9b890a463fd7f4cb17e8e35b3f5448\",  # noqa: E501\n    },\n    \"vits32\": {\n        \"imagenet/classification-v0\": \"f5836e3aff2bab202eaee01d98337a08258159d3b718e0421834e98b3665e10a\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"f3907845eff780a4d29c1c56e0ae053411f02fff6fdce1147c4c3bb2124698cd\",  # noqa: E501\n    },\n    \"vitb32\": {\n        \"imagenet/classification-v0\": \"73025caa78459dc8f9b1de7b58f1d64e24a823f170d17e25fcc8eb6179bea179\",  # noqa: E501\n        \"imagenet/classification-v0-notop\": \"f07b80c03336d731a2a3a02af5cac1e9fc9aa62659cd29e2e7e5c7474150cc71\",  # noqa: E501\n    },\n}","metadata":{"execution":{"iopub.status.busy":"2024-02-14T04:37:13.313959Z","iopub.execute_input":"2024-02-14T04:37:13.314245Z","iopub.status.idle":"2024-02-14T04:37:13.329586Z","shell.execute_reply.started":"2024-02-14T04:37:13.314199Z","shell.execute_reply":"2024-02-14T04:37:13.328763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@keras.saving.register_keras_serializable()\ndef parse_model_inputs(input_shape, input_tensor, **kwargs):\n    if input_tensor is None:\n        return tf.keras.layers.Input(shape=input_shape, **kwargs)\n    else:\n        if not tf.keras.backend.is_keras_tensor(input_tensor):\n            return tf.keras.layers.Input(\n                tensor=input_tensor, shape=input_shape, **kwargs\n            )\n        else:\n            return input_tensor","metadata":{"execution":{"iopub.status.busy":"2024-02-14T04:37:13.331485Z","iopub.execute_input":"2024-02-14T04:37:13.331752Z","iopub.status.idle":"2024-02-14T04:37:13.354094Z","shell.execute_reply.started":"2024-02-14T04:37:13.331725Z","shell.execute_reply":"2024-02-14T04:37:13.353268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_CONFIGS = {\n    \"ViTTiny16\": {\n        \"patch_size\": 16,\n        \"transformer_layer_num\": 12,\n        \"project_dim\": 192,\n        \"mlp_dim\": 768,\n        \"num_heads\": 3,\n        \"mlp_dropout\": 0.0,\n        \"attention_dropout\": 0.0,\n    },\n    \"ViTS16\": {\n        \"patch_size\": 16,\n        \"transformer_layer_num\": 12,\n        \"project_dim\": 384,\n        \"mlp_dim\": 1536,\n        \"num_heads\": 6,\n        \"mlp_dropout\": 0.0,\n        \"attention_dropout\": 0.0,\n    },\n    \"ViTB16\": {\n        \"patch_size\": 16,\n        \"transformer_layer_num\": 12,\n        \"project_dim\": 768,\n        \"mlp_dim\": 3072,\n        \"num_heads\": 12,\n        \"mlp_dropout\": 0.0,\n        \"attention_dropout\": 0.0,\n    },\n    \"ViTL16\": {\n        \"patch_size\": 16,\n        \"transformer_layer_num\": 24,\n        \"project_dim\": 1024,\n        \"mlp_dim\": 4096,\n        \"num_heads\": 16,\n        \"mlp_dropout\": 0.1,\n        \"attention_dropout\": 0.0,\n    },\n    \"ViTH16\": {\n        \"patch_size\": 16,\n        \"transformer_layer_num\": 32,\n        \"project_dim\": 1280,\n        \"mlp_dim\": 5120,\n        \"num_heads\": 16,\n        \"mlp_dropout\": 0.1,\n        \"attention_dropout\": 0.0,\n    },\n    \"ViTTiny32\": {\n        \"patch_size\": 32,\n        \"transformer_layer_num\": 12,\n        \"project_dim\": 192,\n        \"mlp_dim\": 768,\n        \"num_heads\": 3,\n        \"mlp_dropout\": 0.0,\n        \"attention_dropout\": 0.0,\n    },\n    \"ViTS32\": {\n        \"patch_size\": 32,\n        \"transformer_layer_num\": 12,\n        \"project_dim\": 384,\n        \"mlp_dim\": 1536,\n        \"num_heads\": 6,\n        \"mlp_dropout\": 0.0,\n        \"attention_dropout\": 0.0,\n    },\n    \"ViTB32\": {\n        \"patch_size\": 32,\n        \"transformer_layer_num\": 12,\n        \"project_dim\": 768,\n        \"mlp_dim\": 3072,\n        \"num_heads\": 12,\n        \"mlp_dropout\": 0.0,\n        \"attention_dropout\": 0.0,\n    },\n    \"ViTL32\": {\n        \"patch_size\": 32,\n        \"transformer_layer_num\": 24,\n        \"project_dim\": 1024,\n        \"mlp_dim\": 4096,\n        \"num_heads\": 16,\n        \"mlp_dropout\": 0.1,\n        \"attention_dropout\": 0.0,\n    },\n    \"ViTH32\": {\n        \"patch_size\": 32,\n        \"transformer_layer_num\": 32,\n        \"project_dim\": 1280,\n        \"mlp_dim\": 5120,\n        \"num_heads\": 16,\n        \"mlp_dropout\": 0.1,\n        \"attention_dropout\": 0.0,\n    },\n}\n@keras.saving.register_keras_serializable()\nclass ViT(keras.Model):\n    def __init__(\n        self,\n        include_rescaling,\n        include_top,\n        weights=None,\n        input_shape=(None, None, 3),\n        input_tensor=None,\n        pooling=None,\n        num_classes=None,\n        patch_size=None,\n        transformer_layer_num=None,\n        num_heads=None,\n        mlp_dropout=None,\n        attention_dropout=None,\n        activation=None,\n        project_dim=None,\n        mlp_dim=None,\n        classifier_activation=\"softmax\",\n        **kwargs,\n    ):\n        if weights and not tf.io.gfile.exists(weights):\n            raise ValueError(\n                \"The `weights` argument should be either `None` or the path \"\n                \"to the weights file to be loaded. Weights file not found at \"\n                \"location: {weights}\"\n            )\n\n        if include_top and not num_classes:\n            raise ValueError(\n                \"If `include_top` is True, you should specify `num_classes`. \"\n                f\"Received: num_classes={num_classes}\"\n            )\n\n        if include_top and pooling:\n            raise ValueError(\n                f\"`pooling` must be `None` when `include_top=True`.\"\n                f\"Received pooling={pooling} and include_top={include_top}. \"\n            )\n\n        inputs = parse_model_inputs(input_shape, input_tensor)\n        x = inputs\n\n        if include_rescaling:\n            x = layers.Rescaling(1.0 / 255.0, name=\"rescaling\")(x)\n\n        # The previous layer rescales [0..255] to [0..1] if applicable\n        # This one rescales [0..1] to [-1..1] since ViTs expect [-1..1]\n        x = layers.Rescaling(scale=1.0 / 0.5, offset=-1.0, name=\"rescaling_2\")(\n            x\n        )\n\n        encoded_patches = PatchingAndEmbedding(project_dim, patch_size)(x)\n        encoded_patches = layers.Dropout(mlp_dropout)(encoded_patches)\n\n        for _ in range(transformer_layer_num):\n            encoded_patches = TransformerEncoder(\n                project_dim=project_dim,\n                mlp_dim=mlp_dim,\n                num_heads=num_heads,\n                mlp_dropout=mlp_dropout,\n                attention_dropout=attention_dropout,\n                activation=activation,\n            )(encoded_patches)\n\n        output = layers.LayerNormalization(epsilon=1e-6)(encoded_patches)\n\n        if include_top:\n            output = output[:, 0]\n            output = layers.Dense(\n                num_classes, activation=classifier_activation\n            )(output)\n\n        elif pooling == \"token_pooling\":\n            output = output[:, 0]\n        elif pooling == \"avg\":\n            output = layers.GlobalAveragePooling1D()(output)\n\n        # Create model.\n        super().__init__(inputs=inputs, outputs=output, **kwargs)\n\n        if weights is not None:\n            self.load_weights(weights)\n\n        self.include_rescaling = include_rescaling\n        self.include_top = include_top\n        self.input_tensor = input_tensor\n        self.pooling = pooling\n        self.num_classes = num_classes\n        self.patch_size = patch_size\n        self.transformer_layer_num = transformer_layer_num\n        self.num_heads = num_heads\n        self.mlp_dropout = mlp_dropout\n        self.attention_dropout = attention_dropout\n        self.activation = activation\n        self.project_dim = project_dim\n        self.mlp_dim = mlp_dim\n        self.classifier_activation = classifier_activation\n\n    def get_config(self):\n        return {\n            \"include_rescaling\": self.include_rescaling,\n            \"include_top\": self.include_top,\n            \"name\": self.name,\n            \"input_shape\": self.input_shape[1:],\n            \"input_tensor\": self.input_tensor,\n            \"pooling\": self.pooling,\n            \"num_classes\": self.num_classes,\n            \"patch_size\": self.patch_size,\n            \"transformer_layer_num\": self.transformer_layer_num,\n            \"num_heads\": self.num_heads,\n            \"mlp_dropout\": self.mlp_dropout,\n            \"attention_dropout\": self.attention_dropout,\n            \"activation\": self.activation,\n            \"project_dim\": self.project_dim,\n            \"mlp_dim\": self.mlp_dim,\n            \"classifier_activation\": self.classifier_activation,\n            \"trainable\": self.trainable,\n        }\n\n    @classmethod\n    def from_config(cls, config):\n        return cls(**config)\n\n\ndef ViTTiny16(\n    *,\n    include_rescaling,\n    include_top,\n    name=\"ViTTiny16\",\n    weights=None,\n    input_shape=(None, None, 3),\n    input_tensor=None,\n    pooling=None,\n    num_classes=None,\n    activation=keras.activations.gelu,\n    classifier_activation=\"softmax\",\n    **kwargs,\n):\n    \"\"\"Instantiates the ViTTiny16 architecture.\"\"\"\n\n    return ViT(\n        include_rescaling,\n        include_top,\n        name=name,\n        weights=parse_weights(weights, include_top, \"vittiny16\"),\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        pooling=pooling,\n        num_classes=num_classes,\n        patch_size=MODEL_CONFIGS[\"ViTTiny16\"][\"patch_size\"],\n        transformer_layer_num=MODEL_CONFIGS[\"ViTTiny16\"][\n            \"transformer_layer_num\"\n        ],\n        project_dim=MODEL_CONFIGS[\"ViTTiny16\"][\"project_dim\"],\n        mlp_dim=MODEL_CONFIGS[\"ViTTiny16\"][\"mlp_dim\"],\n        num_heads=MODEL_CONFIGS[\"ViTTiny16\"][\"num_heads\"],\n        mlp_dropout=MODEL_CONFIGS[\"ViTTiny16\"][\"mlp_dropout\"],\n        attention_dropout=MODEL_CONFIGS[\"ViTTiny16\"][\"attention_dropout\"],\n        activation=activation,\n        classifier_activation=classifier_activation,\n        **kwargs,\n    )\n\n\ndef ViTS16(\n    *,\n    include_rescaling,\n    include_top,\n    name=\"ViTS16\",\n    weights=None,\n    input_shape=(None, None, 3),\n    input_tensor=None,\n    pooling=None,\n    num_classes=None,\n    activation=keras.activations.gelu,\n    classifier_activation=\"softmax\",\n    **kwargs,\n):\n    \"\"\"Instantiates the ViTS16 architecture.\"\"\"\n\n    return ViT(\n        include_rescaling,\n        include_top,\n        name=name,\n        weights=parse_weights(weights, include_top, \"vits16\"),\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        pooling=pooling,\n        num_classes=num_classes,\n        patch_size=MODEL_CONFIGS[\"ViTS16\"][\"patch_size\"],\n        transformer_layer_num=MODEL_CONFIGS[\"ViTB32\"][\"transformer_layer_num\"],\n        project_dim=MODEL_CONFIGS[\"ViTS16\"][\"project_dim\"],\n        mlp_dim=MODEL_CONFIGS[\"ViTS16\"][\"mlp_dim\"],\n        num_heads=MODEL_CONFIGS[\"ViTS16\"][\"num_heads\"],\n        mlp_dropout=MODEL_CONFIGS[\"ViTS16\"][\"mlp_dropout\"],\n        attention_dropout=MODEL_CONFIGS[\"ViTS16\"][\"attention_dropout\"],\n        activation=activation,\n        classifier_activation=classifier_activation,\n        **kwargs,\n    )\n\n\ndef ViTB16(\n    *,\n    include_rescaling,\n    include_top,\n    name=\"ViTB16\",\n    weights=None,\n    input_shape=(None, None, 3),\n    input_tensor=None,\n    pooling=None,\n    num_classes=None,\n    activation=keras.activations.gelu,\n    classifier_activation=\"softmax\",\n    **kwargs,\n):\n    \"\"\"Instantiates the ViTB16 architecture.\"\"\"\n\n    return ViT(\n        include_rescaling,\n        include_top,\n        name=name,\n        weights=parse_weights(weights, include_top, \"vitb16\"),\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        pooling=pooling,\n        num_classes=num_classes,\n        patch_size=MODEL_CONFIGS[\"ViTB16\"][\"patch_size\"],\n        transformer_layer_num=MODEL_CONFIGS[\"ViTB16\"][\"transformer_layer_num\"],\n        project_dim=MODEL_CONFIGS[\"ViTB16\"][\"project_dim\"],\n        mlp_dim=MODEL_CONFIGS[\"ViTB16\"][\"mlp_dim\"],\n        num_heads=MODEL_CONFIGS[\"ViTB16\"][\"num_heads\"],\n        mlp_dropout=MODEL_CONFIGS[\"ViTB16\"][\"mlp_dropout\"],\n        attention_dropout=MODEL_CONFIGS[\"ViTB16\"][\"attention_dropout\"],\n        activation=activation,\n        classifier_activation=classifier_activation,\n        **kwargs,\n    )\n\n\ndef ViTL16(\n    *,\n    include_rescaling,\n    include_top,\n    name=\"ViTL16\",\n    weights=None,\n    input_shape=(None, None, 3),\n    input_tensor=None,\n    pooling=None,\n    num_classes=None,\n    activation=keras.activations.gelu,\n    classifier_activation=\"softmax\",\n    **kwargs,\n):\n    \"\"\"Instantiates the ViTL16 architecture.\"\"\"\n\n    return ViT(\n        include_rescaling,\n        include_top,\n        name=name,\n        weights=parse_weights(weights, include_top, \"vitl16\"),\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        pooling=pooling,\n        num_classes=num_classes,\n        patch_size=MODEL_CONFIGS[\"ViTL16\"][\"patch_size\"],\n        transformer_layer_num=MODEL_CONFIGS[\"ViTL16\"][\"transformer_layer_num\"],\n        project_dim=MODEL_CONFIGS[\"ViTL16\"][\"project_dim\"],\n        mlp_dim=MODEL_CONFIGS[\"ViTL16\"][\"mlp_dim\"],\n        num_heads=MODEL_CONFIGS[\"ViTL16\"][\"num_heads\"],\n        mlp_dropout=MODEL_CONFIGS[\"ViTL16\"][\"mlp_dropout\"],\n        attention_dropout=MODEL_CONFIGS[\"ViTL16\"][\"attention_dropout\"],\n        activation=activation,\n        classifier_activation=classifier_activation,\n        **kwargs,\n    )\n\n\ndef ViTH16(\n    *,\n    include_rescaling,\n    include_top,\n    name=\"ViTH16\",\n    weights=None,\n    input_shape=(None, None, 3),\n    input_tensor=None,\n    pooling=None,\n    num_classes=None,\n    activation=keras.activations.gelu,\n    classifier_activation=\"softmax\",\n    **kwargs,\n):\n    \"\"\"Instantiates the ViTH16 architecture.\"\"\"\n\n    return ViT(\n        include_rescaling,\n        include_top,\n        name=name,\n        weights=weights,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        pooling=pooling,\n        num_classes=num_classes,\n        patch_size=MODEL_CONFIGS[\"ViTH16\"][\"patch_size\"],\n        transformer_layer_num=MODEL_CONFIGS[\"ViTH16\"][\"transformer_layer_num\"],\n        project_dim=MODEL_CONFIGS[\"ViTH16\"][\"project_dim\"],\n        mlp_dim=MODEL_CONFIGS[\"ViTH16\"][\"mlp_dim\"],\n        num_heads=MODEL_CONFIGS[\"ViTH16\"][\"num_heads\"],\n        mlp_dropout=MODEL_CONFIGS[\"ViTH16\"][\"mlp_dropout\"],\n        attention_dropout=MODEL_CONFIGS[\"ViTH16\"][\"attention_dropout\"],\n        activation=activation,\n        classifier_activation=classifier_activation,\n        **kwargs,\n    )\n\n\ndef ViTTiny32(\n    *,\n    include_rescaling,\n    include_top,\n    name=\"ViTTiny32\",\n    weights=None,\n    input_shape=(None, None, 3),\n    input_tensor=None,\n    pooling=None,\n    num_classes=None,\n    activation=keras.activations.gelu,\n    classifier_activation=\"softmax\",\n    **kwargs,\n):\n    \"\"\"Instantiates the ViTTiny32 architecture.\"\"\"\n\n    return ViT(\n        include_rescaling,\n        include_top,\n        name=name,\n        weights=weights,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        pooling=pooling,\n        num_classes=num_classes,\n        patch_size=MODEL_CONFIGS[\"ViTTiny32\"][\"patch_size\"],\n        transformer_layer_num=MODEL_CONFIGS[\"ViTTiny32\"][\n            \"transformer_layer_num\"\n        ],\n        project_dim=MODEL_CONFIGS[\"ViTTiny32\"][\"project_dim\"],\n        mlp_dim=MODEL_CONFIGS[\"ViTTiny32\"][\"mlp_dim\"],\n        num_heads=MODEL_CONFIGS[\"ViTTiny32\"][\"num_heads\"],\n        mlp_dropout=MODEL_CONFIGS[\"ViTTiny32\"][\"mlp_dropout\"],\n        attention_dropout=MODEL_CONFIGS[\"ViTTiny32\"][\"attention_dropout\"],\n        activation=activation,\n        classifier_activation=classifier_activation,\n        **kwargs,\n    )\n\n\ndef ViTS32(\n    *,\n    include_rescaling,\n    include_top,\n    name=\"ViTS32\",\n    weights=None,\n    input_shape=(None, None, 3),\n    input_tensor=None,\n    pooling=None,\n    num_classes=None,\n    activation=keras.activations.gelu,\n    classifier_activation=\"softmax\",\n    **kwargs,\n):\n    \"\"\"Instantiates the ViTS32 architecture.\"\"\"\n\n    return ViT(\n        include_rescaling,\n        include_top,\n        name=name,\n        weights=parse_weights(weights, include_top, \"vits32\"),\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        pooling=pooling,\n        num_classes=num_classes,\n        patch_size=MODEL_CONFIGS[\"ViTS32\"][\"patch_size\"],\n        transformer_layer_num=MODEL_CONFIGS[\"ViTS32\"][\"transformer_layer_num\"],\n        project_dim=MODEL_CONFIGS[\"ViTS32\"][\"project_dim\"],\n        mlp_dim=MODEL_CONFIGS[\"ViTS32\"][\"mlp_dim\"],\n        num_heads=MODEL_CONFIGS[\"ViTS32\"][\"num_heads\"],\n        mlp_dropout=MODEL_CONFIGS[\"ViTS32\"][\"mlp_dropout\"],\n        attention_dropout=MODEL_CONFIGS[\"ViTS32\"][\"attention_dropout\"],\n        activation=activation,\n        classifier_activation=classifier_activation,\n        **kwargs,\n    )\n\n\ndef ViTB32(\n    *,\n    include_rescaling,\n    include_top,\n    name=\"ViTB32\",\n    weights=None,\n    input_shape=(None, None, 3),\n    input_tensor=None,\n    pooling=None,\n    num_classes=None,\n    activation=keras.activations.gelu,\n    classifier_activation=\"softmax\",\n    **kwargs,\n):\n    \"\"\"Instantiates the ViTB32 architecture.\"\"\"\n\n    return ViT(\n        include_rescaling,\n        include_top,\n        name=name,\n        weights=parse_weights(weights, include_top, \"vitb32\"),\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        pooling=pooling,\n        num_classes=num_classes,\n        patch_size=MODEL_CONFIGS[\"ViTB32\"][\"patch_size\"],\n        transformer_layer_num=MODEL_CONFIGS[\"ViTB32\"][\"transformer_layer_num\"],\n        project_dim=MODEL_CONFIGS[\"ViTB32\"][\"project_dim\"],\n        mlp_dim=MODEL_CONFIGS[\"ViTB32\"][\"mlp_dim\"],\n        num_heads=MODEL_CONFIGS[\"ViTB32\"][\"num_heads\"],\n        mlp_dropout=MODEL_CONFIGS[\"ViTB32\"][\"mlp_dropout\"],\n        attention_dropout=MODEL_CONFIGS[\"ViTB32\"][\"attention_dropout\"],\n        activation=activation,\n        classifier_activation=classifier_activation,\n        **kwargs,\n    )\n\n\ndef ViTL32(\n    *,\n    include_rescaling,\n    include_top,\n    name=\"ViTL32\",\n    weights=None,\n    input_shape=(None, None, 3),\n    input_tensor=None,\n    pooling=None,\n    num_classes=None,\n    activation=keras.activations.gelu,\n    classifier_activation=\"softmax\",\n    **kwargs,\n):\n    \"\"\"Instantiates the ViTL32 architecture.\"\"\"\n\n    return ViT(\n        include_rescaling,\n        include_top,\n        name=name,\n        weights=weights,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        pooling=pooling,\n        num_classes=num_classes,\n        patch_size=MODEL_CONFIGS[\"ViTL32\"][\"patch_size\"],\n        transformer_layer_num=MODEL_CONFIGS[\"ViTL32\"][\"transformer_layer_num\"],\n        project_dim=MODEL_CONFIGS[\"ViTL32\"][\"project_dim\"],\n        mlp_dim=MODEL_CONFIGS[\"ViTL32\"][\"mlp_dim\"],\n        num_heads=MODEL_CONFIGS[\"ViTL32\"][\"num_heads\"],\n        mlp_dropout=MODEL_CONFIGS[\"ViTL32\"][\"mlp_dropout\"],\n        attention_dropout=MODEL_CONFIGS[\"ViTL32\"][\"attention_dropout\"],\n        activation=activation,\n        classifier_activation=classifier_activation,\n        **kwargs,\n    )\n\n\ndef ViTH32(\n    *,\n    include_rescaling,\n    include_top,\n    name=\"ViTH32\",\n    weights=None,\n    input_shape=(None, None, 3),\n    input_tensor=None,\n    pooling=None,\n    num_classes=None,\n    activation=keras.activations.gelu,\n    classifier_activation=\"softmax\",\n    **kwargs,\n):\n    \"\"\"Instantiates the ViTH32 architecture.\"\"\"\n\n    return ViT(\n        include_rescaling,\n        include_top,\n        name=name,\n        weights=weights,\n        input_shape=input_shape,\n        input_tensor=input_tensor,\n        pooling=pooling,\n        num_classes=num_classes,\n        patch_size=MODEL_CONFIGS[\"ViTH32\"][\"patch_size\"],\n        transformer_layer_num=MODEL_CONFIGS[\"ViTH32\"][\"transformer_layer_num\"],\n        project_dim=MODEL_CONFIGS[\"ViTH32\"][\"project_dim\"],\n        mlp_dim=MODEL_CONFIGS[\"ViTH32\"][\"mlp_dim\"],\n        num_heads=MODEL_CONFIGS[\"ViTH32\"][\"num_heads\"],\n        mlp_dropout=MODEL_CONFIGS[\"ViTH32\"][\"mlp_dropout\"],\n        attention_dropout=MODEL_CONFIGS[\"ViTH32\"][\"attention_dropout\"],\n        activation=activation,\n        classifier_activation=classifier_activation,\n        **kwargs,\n    )\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-14T04:37:13.355438Z","iopub.execute_input":"2024-02-14T04:37:13.355716Z","iopub.status.idle":"2024-02-14T04:37:13.409876Z","shell.execute_reply.started":"2024-02-14T04:37:13.355687Z","shell.execute_reply":"2024-02-14T04:37:13.408993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapping = {}\nimages = []\n\nwith open('/kaggle/input/imagenet-object-localization-challenge/LOC_synset_mapping.txt', 'r') as f:\n    lines = f.readlines()\n    for line in lines:\n        mapping[line.split()[0]] = ' '.join(line.split(',')[0].split()[1:])\n        \nwith open('/kaggle/input/imagenet-object-localization-challenge/ILSVRC/ImageSets/CLS-LOC/train_cls.txt', 'r') as f:\n    lines = f.readlines()\n    for line in lines:\n        images.append(line.split()[0])\n\nbasedir = \"/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC/train/\"\n\nprint(mapping[images[500939].split('/')[0]])\n\n\n\ntrain_set = tf.keras.utils.image_dataset_from_directory(basedir,image_size=(224, 224),batch_size=16 * 8)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T04:37:13.411215Z","iopub.execute_input":"2024-02-14T04:37:13.411541Z","iopub.status.idle":"2024-02-14T04:38:48.856986Z","shell.execute_reply.started":"2024-02-14T04:37:13.41151Z","shell.execute_reply":"2024-02-14T04:38:48.855818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PlotLearning(tf.keras.callbacks.Callback):\n    \"\"\"\n    Callback to plot the learning curves of the model during training.\n    \"\"\"\n    def on_train_begin(self, logs={}):\n        self.metrics = {}\n        for metric in logs:\n            self.metrics[metric] = []\n\n    def on_train_batch_end(self, batch, logs={}):\n        # Storing metrics\n        for metric in logs:\n            if metric in self.metrics:\n                self.metrics[metric].append(logs.get(metric))\n            else:\n                self.metrics[metric] = [logs.get(metric)]\n        \n        if(batch%200==0):\n            with open('/kaggle/working/my_list.pkl', 'wb') as file:\n                pickle.dump(self.metrics, file)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T04:38:48.85858Z","iopub.execute_input":"2024-02-14T04:38:48.858855Z","iopub.status.idle":"2024-02-14T04:38:48.864853Z","shell.execute_reply.started":"2024-02-14T04:38:48.858826Z","shell.execute_reply":"2024-02-14T04:38:48.863969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import  EarlyStopping,ModelCheckpoint\ncb = [EarlyStopping(patience=3, monitor='sparse_categorical_accuracy', mode='auto' ,restore_best_weights=True,min_delta = 0.01,verbose = True),\n     PlotLearning()]\nwith tpu_strategy.scope():\n\n    vit=ViTTiny16(\n        include_rescaling=True,\n        include_top=True,\n        num_classes=1000,\n        classifier_activation='softmax',\n        input_shape=(224,224,3),\n        weights=None\n        )\n\n    vit.compile(optimizer=\"adam\",loss = \"sparse_categorical_crossentropy\",metrics=['sparse_categorical_accuracy'])\n\nvit.fit(train_set,callbacks = cb ,verbose = True, epochs = 90 ,batch_size=16*8)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T04:42:26.062737Z","iopub.execute_input":"2024-02-14T04:42:26.063046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomModel(tf.keras.Model):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.loss_tracker = tf.keras.metrics.SparseCategoricalCrossentropy(name=\"loss\")\n#         self.mae_metric = tf.keras.metrics.MeanAbsoluteError(name=\"mae\")\n        self.sparscat_metric=tf.keras.metrics.SparseCategoricalAccuracy(name='sparse_categorical_accuracy')\n    \n    def train_step(self, data):\n        # Unpack the data. Its structure depends on your model and\n        # on what you pass to `fit()`.\n        x, y = data\n        print('.')\n        true_x=x\n        eps=0.3\n        rand_minmax = eps\n        norm=np.inf\n        nb_iter=5\n        eps_iter=0.08\n        eta = tf.zeros_like(x)\n\n        # Clip eta\n        eta = clip_eta(eta, norm, eps)\n        x = true_x + eta\n\n        i = 0\n        result={}\n\n        while i < nb_iter:\n            with tf.GradientTape(persistent=True) as tape:\n                tape.watch(x)\n                predictions = self(x,training=True)\n                loss=self.compute_loss(y=y,y_pred=predictions)\n\n            gradients = tape.gradient(loss, self.trainable_variables)\n            adv_grad = tape.gradient(loss, x)\n            del tape\n            self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))\n            print(y)\n            for metric in self.metrics:\n                metric.update_state(y, predictions)\n            # Return a dict mapping metric names to current value\n            result= {m.name: m.result() for m in self.metrics}\n\n            x = self.fast_gradient_method(\n                grad=adv_grad,\n                x=x,\n                eps=eps_iter,\n                norm=norm,\n                y=y\n            )\n\n            # Clipping perturbation eta to norm norm ball\n            eta = x - true_x\n            eta = clip_eta(eta, norm, eps)\n            x = true_x + eta\n\n\n            i += 1\n        return result\n\n    def fast_gradient_method(self, grad, x, eps, norm, y,):\n        # cast to tensor if provided as numpy array\n        x = tf.cast(x, tf.float32)\n\n        optimal_perturbation = optimize_linear(grad, eps, norm)\n        # Add perturbation to original example to obtain adversarial example\n        adv_x = x + optimal_perturbation\n\n        return adv_x\n    @property\n    def metrics(self):\n        # We list our `Metric` objects here so that `reset_states()` can be\n        # called automatically at the start of each epoch\n        # or at the start of `evaluate()`.\n        # If you don't implement this property, you have to call\n        # `reset_states()` yourself at the time of your choosing.\n        return [ self.loss_tracker,self.sparscat_metric]\n    def test_step(self, data):\n        # Unpack the data\n        print('.')\n        x, y = data\n        \n        adv_x=projected_gradient_descent(self,x,0.3,0.02,20,np.inf,y=y)\n        # Compute predictions\n        y_pred = self(adv_x, training=False)\n        # Updates the metrics tracking the loss\n        self.compute_loss(y=y, y_pred=y_pred)\n        # Update the metrics.\n        for metric in self.metrics:\n                metric.update_state(y, y_pred)\n        # Return a dict mapping metric names to current value.\n        # Note that it will include the loss (tracked in self.metrics).\n        return {m.name: m.result() for m in self.metrics}\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T04:38:49.599564Z","iopub.status.idle":"2024-02-14T04:38:49.599935Z","shell.execute_reply.started":"2024-02-14T04:38:49.599766Z","shell.execute_reply":"2024-02-14T04:38:49.599784Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with tpu_strategy.scope():\n\n#     vit=ViTTiny16(\n#         include_rescaling=True,\n#         include_top=True,\n#         num_classes=1000,\n#         classifier_activation='softmax',\n#         input_shape=(224,224,3),\n#         weights='imagenet'\n#         )\n\n#     vit.compile(optimizer=\"adam\",loss = \"sparse_categorical_crossentropy\",metrics=['sparse_categorical_accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2024-02-14T04:38:49.60119Z","iopub.status.idle":"2024-02-14T04:38:49.601563Z","shell.execute_reply.started":"2024-02-14T04:38:49.601397Z","shell.execute_reply":"2024-02-14T04:38:49.601416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vit.evaluate(train_set)","metadata":{"execution":{"iopub.status.busy":"2024-02-14T04:38:49.6029Z","iopub.status.idle":"2024-02-14T04:38:49.603324Z","shell.execute_reply.started":"2024-02-14T04:38:49.603069Z","shell.execute_reply":"2024-02-14T04:38:49.603086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}