{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf \nimport random, numpy as np, os\ndef config_gpu(mp=False):\n    print('Eager Model : ', tf.executing_eagerly())\n    print('TensorFlow Cuda Built Test : ', tf.test.is_built_with_cuda)\n    print('TensorFlow GPU Detected : ', tf.test.gpu_device_name())\n    print('TensorFlow System Cuda Version : ', tf.sysconfig.get_build_info()[\"cuda_version\"])\n    print('TensorFlow System CudNN Version : ', tf.sysconfig.get_build_info()[\"cudnn_version\"] )\n\n    AUTO = tf.data.AUTOTUNE\n    GPUS = tf.config.list_physical_devices('GPU')\n    if GPUS:\n        try:\n            for GPU in GPUS:\n                tf.config.experimental.set_memory_growth(GPU, True)\n                logical_gpus = tf.config.list_logical_devices('GPU')\n                print(len(GPUS), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\") \n        except RuntimeError as  RE:\n            print(RE)\n    if mp:\n        tf.keras.mixed_precision.set_global_policy('mixed_float16')\n        print('Mixed precision enabled')\n        \ntf.random.set_seed(100)\nconfig_gpu(mp=False)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE  = 12\nIMG_SIZE    = 256\nCHANNELS    = 3\nEPOCHS = 10\nLR = 0.002","metadata":{"execution":{"iopub.status.busy":"2022-05-24T11:09:48.295841Z","iopub.execute_input":"2022-05-24T11:09:48.296103Z","iopub.status.idle":"2022-05-24T11:09:48.300952Z","shell.execute_reply.started":"2022-05-24T11:09:48.296069Z","shell.execute_reply":"2022-05-24T11:09:48.299769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf \nfrom tensorflow import keras \nfrom tensorflow.keras import layers\n\n# Multi-Atrous Branch\nclass MultiAtrous(keras.Model):\n    def __init__(self, \n                 dilation_rates=[6, 12, 18], \n                 upsampling=1, \n                 kernel_size=3, \n                 padding=\"same\",  **kwargs):\n        super(MultiAtrous, self).__init__(name='MultiAtrous', **kwargs)\n        self.dilation_rates = dilation_rates\n        self.kernel_size = kernel_size \n        self.upsampling = upsampling\n        self.padding = padding\n      \n        # Dilated Convolutions                     \n        self.dilated_convs = [\n                                layers.Conv2D(\n                                    filters       = int(1024 / 4), \n                                    kernel_size   = self.kernel_size,  \n                                    padding       = self.padding, \n                                    dilation_rate = rate\n                                ) for rate in self.dilation_rates\n                             ]\n        \n        # Global Average Pooling Branch \n        self.gap_branch = keras.Sequential(\n            [\n                layers.GlobalAveragePooling2D(keepdims=True),\n                layers.Conv2D(int(1024 / 2), kernel_size=1),\n                layers.Activation('relu'),\n                layers.UpSampling2D(size=self.upsampling, interpolation=\"bilinear\")\n            ] , name='gap_branch'\n        )\n        \n    def call(self, inputs, training=None, **kwargs):\n        local_feature = []\n\n        for dilated_conv in self.dilated_convs:\n            x = dilated_conv(inputs) \n            x = self.gap_branch(x)\n            local_feature.append(x)\n            \n        return tf.concat(local_feature, axis=-1)\n    def get_config(self):\n        config = {\n            'dilation_rates': self.dilation_rates,\n            'kernel_size'   : self.kernel_size,\n            'padding'       : self.padding,\n            'upsampling'    : self.upsampling\n        }\n        base_config = super(MultiAtrous, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n    \n# DOLG: Local-Branch\nclass DOLGLocalBranch(keras.Model):\n    def __init__(self, IMG_SIZE, **kwargs):\n        super(DOLGLocalBranch, self).__init__(name='LocalBranch', **kwargs)\n        self.multi_atrous = MultiAtrous(padding='same', upsampling=int(IMG_SIZE/32))\n        self.conv1 = layers.Conv2D(1024, kernel_size=1)\n        self.conv2 = layers.Conv2D(1024, kernel_size=1, use_bias=False)\n        self.conv3 = layers.Conv2D(1024, kernel_size=1)\n        self.bn = layers.BatchNormalization()\n\n    def call(self, inputs, training=None, **kwargs):\n        # Local Branach + Normalization / Conv-Bn Module \n        local_feat = self.multi_atrous(inputs)\n        local_feat = self.conv1(local_feat)\n        local_feat = tf.nn.relu(local_feat)\n        \n        # Self-Attention\n        local_feat = self.conv2(local_feat)\n        local_feat = self.bn(local_feat)\n\n        # l-2 norms\n        norm_local_feat = tf.math.l2_normalize(local_feat)\n\n        # softplus activations\n        attn_map = tf.nn.relu(local_feat)\n        attn_map = self.conv3(attn_map)\n        attn_map = keras.activations.softplus(attn_map) \n\n        # Output of the Local-Branch \n        return  norm_local_feat * attn_map ","metadata":{"execution":{"iopub.status.busy":"2022-05-24T11:09:48.304576Z","iopub.execute_input":"2022-05-24T11:09:48.304861Z","iopub.status.idle":"2022-05-24T11:09:48.325565Z","shell.execute_reply.started":"2022-05-24T11:09:48.304825Z","shell.execute_reply":"2022-05-24T11:09:48.324586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class OrthogonalFusion(layers.Layer):\n    def __init__(self, **kwargs):\n        super().__init__(name='OrthogonalFusion', **kwargs)\n    def call(self, inputs):\n        local_feat, global_feat = inputs\n        height = local_feat.shape[1]\n        width  = local_feat.shape[2]\n        depth  = local_feat.shape[3]\n    \n        local_feat = tf.reshape(local_feat, [-1, height*width, depth])\n        local_feat = tf.transpose(local_feat, perm=[0, 2, 1])\n        \n        projection = tf.matmul(\n            tf.expand_dims(global_feat, axis=1), \n            local_feat\n        )\n        projection = tf.matmul(\n            tf.expand_dims(global_feat, axis=2),\n            projection\n        )\n        projection = tf.reshape(projection, [-1, height, width, depth])\n        \n        global_feat_norm = tf.norm(global_feat, ord=2, axis=1)  \n        projection = projection / tf.reshape(global_feat_norm*global_feat_norm, shape=[-1, 1, 1, 1])\n        local_feat = tf.transpose(local_feat, perm=[0, 1, 2])\n        local_feat = tf.reshape(local_feat, [-1, height, width, depth])\n    \n        orthogonal_comp = local_feat - projection\n        global_feat = tf.expand_dims(tf.expand_dims(global_feat, axis=1), axis=1)\n        global_feat = tf.broadcast_to(global_feat, tf.shape(local_feat))\n        output = tf.concat([global_feat, orthogonal_comp], axis=-1)\n        return output\n","metadata":{"execution":{"iopub.status.busy":"2022-05-24T11:09:48.327508Z","iopub.execute_input":"2022-05-24T11:09:48.32799Z","iopub.status.idle":"2022-05-24T11:09:48.340759Z","shell.execute_reply.started":"2022-05-24T11:09:48.327951Z","shell.execute_reply":"2022-05-24T11:09:48.339807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GeneralizedMeanPooling2D(layers.Layer):\n    def __init__(self, init_norm=3.0, normalize=False, epsilon=1e-6, **kwargs):\n        self.init_norm = init_norm\n        self.normalize = normalize\n        self.epsilon   = epsilon\n        super(GeneralizedMeanPooling2D, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.p = self.add_weight(name=\"norms\", \n                                 shape=(input_shape[-1],),\n                                 initializer=keras.initializers.constant(self.init_norm),\n                                 trainable=True)\n        super(GeneralizedMeanPooling2D, self).build(input_shape)\n\n    def call(self, inputs):\n        x = tf.abs(tf.maximum(self.epsilon, inputs))\n        x = tf.pow(x, self.p)\n        x = tf.reduce_mean(x, axis=[1,2], keepdims=False) \n        x = tf.pow(x, (1.0 / self.p))\n        if self.normalize:\n            x = tf.nn.l2_normalize(x, 1)\n        return x\n\n    def get_config(self):\n        config = {\n            'init_norm' : self.init_norm,\n            'normalize' : self.normalize,\n            'epsilon'   : self.epsilon\n        }\n        base_config = super(GeneralizedMeanPooling2D, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","metadata":{"execution":{"iopub.status.busy":"2022-05-24T11:09:48.342352Z","iopub.execute_input":"2022-05-24T11:09:48.342641Z","iopub.status.idle":"2022-05-24T11:09:48.354989Z","shell.execute_reply.started":"2022-05-24T11:09:48.342606Z","shell.execute_reply":"2022-05-24T11:09:48.3542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base = keras.applications.ResNet50(\n            include_top=False,\n            weights='imagenet',\n            input_tensor=Input((IMG_SIZE, IMG_SIZE, 3))\n        )\n\nnew_base = keras.Model(\n    [base.inputs], \n    [\n        base.get_layer('conv3_block4_out').output,  # fol local branch \n        base.get_layer('conv5_block3_out').output   # for global branch \n        ], \n    name='ResNet'\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T11:09:48.4094Z","iopub.execute_input":"2022-05-24T11:09:48.410161Z","iopub.status.idle":"2022-05-24T11:09:52.73656Z","shell.execute_reply.started":"2022-05-24T11:09:48.410122Z","shell.execute_reply":"2022-05-24T11:09:52.735793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DOLGNet(keras.Model):\n    def __init__(self, backbone=None, num_classes=1, activation=None, **kwargs):\n        super(DOLGNet, self).__init__(name='DOLGNet', **kwargs)\n        # Number of classes \n        self.num_classes = num_classes\n        self.activation  = activation\n        \n        # Base blcoks \n        self.base = backbone\n        self.base_input_shape  = self.base.input_shape[0][1]\n\n        # Top building blocks \n        self.orthogonal_fusion = OrthogonalFusion()\n        self.local_branch      = DOLGLocalBranch(IMG_SIZE=self.base_input_shape)\n        \n        # Tail blcok 1 \n        self.glob_branch_pool = keras.Sequential(\n            [\n                GeneralizedMeanPooling2D(),\n                layers.Dense(1024, activation=None)\n            ], \n            name='GlobalBranchPooling'\n        )\n        \n        # Head block\n        self.classifier = keras.Sequential(\n            [\n                layers.GlobalAveragePooling2D(name='HeadGAP'),\n                layers.Dense(self.num_classes, activation = self.activation)\n            ], \n            name='Classifiers'\n        )\n       \n    # forwarding the computation \n    def call(self, inputs, training=None, **kwargs):\n        # Get tensor from target layers \n        to_local, to_global = self.base(inputs)\n\n        # Pass the received tensor to Top building blocks \n        local_feat      = self.local_branch(to_local)\n        global_feat     = self.glob_branch_pool(to_global)\n        orthogonal_feat = self.orthogonal_fusion([local_feat, global_feat]) \n        return self.classifier(orthogonal_feat)\n\n    def build_graph(self):\n        x = keras.Input(shape=(self.base_input_shape, self.base_input_shape, 3))\n        return keras.Model(inputs=[x], outputs=self.call(x))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-24T11:09:52.738505Z","iopub.execute_input":"2022-05-24T11:09:52.7391Z","iopub.status.idle":"2022-05-24T11:09:52.760006Z","shell.execute_reply.started":"2022-05-24T11:09:52.739063Z","shell.execute_reply":"2022-05-24T11:09:52.759346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_CLASSES=100\ndolg_net = DOLGNet(new_base, num_classes=NUM_CLASSES, activation='softmax')\ndolg_net.build_graph().summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T11:09:52.761261Z","iopub.execute_input":"2022-05-24T11:09:52.761749Z","iopub.status.idle":"2022-05-24T11:09:55.113038Z","shell.execute_reply.started":"2022-05-24T11:09:52.761673Z","shell.execute_reply":"2022-05-24T11:09:55.112339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(tf.keras.utils.plot_model(dolg_net.build_graph(), \n                                  show_shapes=True, show_layer_names=True, expand_nested=False))","metadata":{"execution":{"iopub.status.busy":"2022-05-24T11:09:55.114434Z","iopub.execute_input":"2022-05-24T11:09:55.114698Z","iopub.status.idle":"2022-05-24T11:09:57.453334Z","shell.execute_reply.started":"2022-05-24T11:09:55.114663Z","shell.execute_reply":"2022-05-24T11:09:57.452505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\nimport os\n\ninput_dir = os.path.join('..', 'input')\ndata_dir = os.path.join(input_dir, 'landmark-recognition-2021')\ntrain_label_info = os.path.join(data_dir, 'train.csv')\ntrain_dir = os.path.join(data_dir, 'train')\ntest_dir = os.path.join(data_dir, 'test')\ntraincsv = pd.read_csv(train_label_info)\ntraincsv['path'] = traincsv['images'].transform(lambda x: train_dir+'/'+str(x[0])+'/'+str(x[1])+'/'+\n                                            str(x[2])+'/'+str(x)+'.jpg')\ntopten = traincsv['landmark_id'].value_counts().head(100).index\ndftrain = traincsv.loc[traincsv['landmark_id'].isin(topten)].copy()\nfrom sklearn.preprocessing import LabelEncoder\nencoder = LabelEncoder()\ndftrain['label'] = encoder.fit_transform(dftrain['landmark_id'])\nfrom sklearn.model_selection import train_test_split\ntrain, val = train_test_split(dftrain, test_size=0.2)\ndatagen = tf.keras.preprocessing.image.ImageDataGenerator(\n            rescale=1./255,\n            rotation_range=20,\n            width_shift_range=0.2,\n            height_shift_range=0.2,\n            horizontal_flip=True)\n    \n\ntrain_data = datagen.flow_from_dataframe(train, x_col='path', y_col='label', class_mode='raw',\n                                             target_size=(256,256))\n\nval_data = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255.).flow_from_dataframe(\n                    val, x_col='path', y_col='label', class_mode='raw', target_size=(256,256))","metadata":{"execution":{"iopub.status.busy":"2022-05-25T03:33:58.275106Z","iopub.execute_input":"2022-05-25T03:33:58.275433Z","iopub.status.idle":"2022-05-25T03:33:58.98592Z","shell.execute_reply.started":"2022-05-25T03:33:58.275354Z","shell.execute_reply":"2022-05-25T03:33:58.984411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import optimizers, metrics, losses\n\ndolg_net.compile(\n    optimizer='Adam',\n    loss='sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy'])\ndolg_net.fit(train_data, epochs=10, validation_data=val_data, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T08:08:36.131086Z","iopub.execute_input":"2022-05-24T08:08:36.131341Z"},"trusted":true},"execution_count":null,"outputs":[]}]}