{"cells":[{"metadata":{},"cell_type":"markdown","source":"#### Work in progress\n\nFeel free to give any feedback!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nimport math\nimport pandas as pd\nfrom sklearn import model_selection\nimport glob\nimport os\nfrom zipfile import ZipFile\nimport shutil\nimport tqdm.notebook as tqdm\n\nimport logging\ntf.get_logger().setLevel(logging.ERROR)\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ngpus = tf.config.experimental.list_physical_devices('GPU')\nnum_gpus = len(gpus)\nif gpus:\n    try:\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n        print(num_gpus, \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        print(e)\n\n#     policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n#     tf.keras.mixed_precision.experimental.set_policy(policy)\n#     print('Compute dtype: %s' % policy.compute_dtype)\n#     print('Variable dtype: %s' % policy.variable_dtype)\n\n    \nif num_gpus == 0:\n    strategy = tf.distribute.OneDeviceStrategy(device='CPU')\n    print(\"Setting strategy to OneDeviceStrategy(device='CPU')\")\nelif num_gpus == 1:\n    strategy = tf.distribute.OneDeviceStrategy(device='GPU')\n    print(\"Setting strategy to OneDeviceStrategy(device='GPU')\")\nelse:\n    strategy = tf.distribute.MirroredStrategy()\n    print(\"Setting strategy to MirroredStrategy()\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"config = {\n    'learning_rate': 1e-3,\n    'momentum': 0.9,\n    'scale': 30,\n    'margin': 0.1,\n    'clip_grad': 10.0,\n    'n_epochs': 30,\n    'batch_size': 32,\n    'input_size': (384, 384, 3),\n    'n_classes': 81313,\n    'dense_units': 1024,\n    'dropout_rate': 0.1,\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_submission_file(input_path, alpha=0.5):\n    files_paths = glob.glob(input_path + 'test/*/*/*/*')\n    mapping = {}\n    for path in files_paths:\n        mapping[path.split('/')[-1].split('.')[0]] = path\n    df = pd.read_csv(input_path + 'sample_submission.csv')\n    df['path'] = df['id'].map(mapping)\n    df['label'] = -1\n    df['prob'] = -1\n    return df\n\ndef read_train_file(input_path, alpha=0.5):\n    files_paths = glob.glob(input_path + 'train/*/*/*/*')\n    mapping = {}\n    for path in files_paths:\n        mapping[path.split('/')[-1].split('.')[0]] = path\n    df = pd.read_csv(input_path + 'train.csv')\n    df['path'] = df['id'].map(mapping)\n    \n    counts_map = dict(\n        df.groupby('landmark_id')['path'].agg(lambda x: len(x)))\n    df['counts'] = df['landmark_id'].map(counts_map)\n    df['prob'] = (\n        (1/df.counts**alpha) / (1/df.counts**alpha).max()).astype(np.float32)\n    uniques = df['landmark_id'].unique()\n    df['label'] = df['landmark_id'].map(dict(zip(uniques, range(len(uniques)))))\n    return df, dict(zip(range(len(uniques)), uniques))\n\n\nsubmission_df = read_submission_file('../input/landmark-recognition-2020/')\ntrain_df, mapping = read_train_file('../input/landmark-recognition-2020/')\ntrain_df.head(10)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _get_transform_matrix(rotation, shear, hzoom, wzoom, hshift, wshift):\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n\n    # convert degrees to radians\n    rotation = math.pi * rotation / 360.\n    shear    = math.pi * shear    / 360.\n\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    rot_mat = get_3x3_mat([c1,    s1,   zero ,\n                           -s1,   c1,   zero ,\n                           zero,  zero, one ])\n\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_mat = get_3x3_mat([one,  s2,   zero ,\n                             zero, c2,   zero ,\n                             zero, zero, one ])\n\n    zoom_mat = get_3x3_mat([one/hzoom, zero,      zero,\n                            zero,      one/wzoom, zero,\n                            zero,      zero,      one])\n\n    shift_mat = get_3x3_mat([one,  zero, hshift,\n                             zero, one,  wshift,\n                             zero, zero, one   ])\n\n    return tf.matmul(\n        tf.matmul(rot_mat, shear_mat),\n        tf.matmul(zoom_mat, shift_mat)\n    )\n\ndef _spatial_transform(image,\n                       rotation=3.0,\n                       shear=2.0,\n                       hzoom=8.0,\n                       wzoom=8.0,\n                       hshift=8.0,\n                       wshift=8.0):\n\n    ydim = tf.gather(tf.shape(image), 0)\n    xdim = tf.gather(tf.shape(image), 1)\n    xxdim = xdim % 2\n    yxdim = ydim % 2\n\n    # random rotation, shear, zoom and shift\n    rotation = rotation * tf.random.normal([1], dtype='float32')\n    shear = shear * tf.random.normal([1], dtype='float32')\n    hzoom = 1.0 + tf.random.normal([1], dtype='float32') / hzoom\n    wzoom = 1.0 + tf.random.normal([1], dtype='float32') / wzoom\n    hshift = hshift * tf.random.normal([1], dtype='float32')\n    wshift = wshift * tf.random.normal([1], dtype='float32')\n\n    m = _get_transform_matrix(\n        rotation, shear, hzoom, wzoom, hshift, wshift)\n\n    # origin pixels\n    y = tf.repeat(tf.range(ydim//2, -ydim//2,-1), xdim)\n    x = tf.tile(tf.range(-xdim//2, xdim//2), [ydim])\n    z = tf.ones([ydim*xdim], dtype='int32')\n    idx = tf.stack([y, x, z])\n\n    # destination pixels\n    idx2 = tf.matmul(m, tf.cast(idx, dtype='float32'))\n    idx2 = tf.cast(idx2, dtype='int32')\n    # clip to origin pixels range\n    idx2y = tf.clip_by_value(idx2[0,], -ydim//2+yxdim+1, ydim//2)\n    idx2x = tf.clip_by_value(idx2[1,], -xdim//2+xxdim+1, xdim//2)\n    idx2 = tf.stack([idx2y, idx2x, idx2[2,]])\n\n    # apply destinations pixels to image\n    idx3 = tf.stack([ydim//2-idx2[0,], xdim//2-1+idx2[1,]])\n    d = tf.gather_nd(image, tf.transpose(idx3))\n    image = tf.reshape(d, [ydim, xdim, 3])\n    return image\n\ndef _pixel_transform(image,\n                     saturation_delta=0.3,\n                     contrast_delta=0.1,\n                     brightness_delta=0.2):\n    image = tf.image.random_saturation(\n        image, 1-saturation_delta, 1+saturation_delta)\n    image = tf.image.random_contrast(\n        image, 1-contrast_delta, 1+contrast_delta)\n    image = tf.image.random_brightness(\n        image, brightness_delta)\n    return image\n\ndef preprocess_input(image, target_size, augment=False):\n    \n    image = tf.image.resize(\n        image, target_size, method='bilinear')\n\n    image = tf.cast(image, tf.uint8)\n    if augment:\n        image = _spatial_transform(image)\n        image = _pixel_transform(image)\n    image = tf.cast(image, tf.float32)\n    image /= 255.\n    return image\n\ndef create_dataset(df, training, batch_size, input_size):\n\n    def read_image(image_path):\n        image = tf.io.read_file(image_path)\n        return tf.image.decode_jpeg(image, channels=3)\n    \n    def filter_by_probs(x, y, p):\n        if p > np.random.uniform(0, 1):\n            return True\n        return False\n\n    image_paths, labels, probs = df.path, df.label, df.prob\n\n    dataset = tf.data.Dataset.from_tensor_slices((image_paths, labels, probs))\n    if training:\n        dataset = dataset.shuffle(100_000)\n    dataset = dataset.map(\n        lambda x, y, p: (read_image(x), y, p),\n        tf.data.experimental.AUTOTUNE)\n    if training:\n        dataset = dataset.filter(filter_by_probs)\n    dataset = dataset.map(\n        lambda x, y, p: (preprocess_input(x, input_size[:2], training), y),\n        tf.data.experimental.AUTOTUNE)\n    dataset = dataset.batch(batch_size)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output\n\n\ndef create_model(input_shape,\n                 n_classes,\n                 dense_units=512,\n                 dropout_rate=0.1,\n                 scale=30,\n                 margin=0.3):\n\n    backbone = tf.keras.applications.ResNet50(\n        include_top=False,\n        input_shape=input_shape,\n        weights=('../input/imagenet-weights/' +\n                 'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n    )\n\n    pooling = tf.keras.layers.GlobalAveragePooling2D(name='head/pooling')\n    dropout = tf.keras.layers.Dropout(dropout_rate, name='head/dropout')\n    dense = tf.keras.layers.Dense(dense_units, name='head/dense')\n\n    margin = ArcMarginProduct(\n        n_classes=n_classes,\n        s=scale,\n        m=margin,\n        name='head/arc_margin',\n        dtype='float32')\n\n    softmax = tf.keras.layers.Softmax(dtype='float32')\n\n    image = tf.keras.layers.Input(input_shape, name='input/image')\n    label = tf.keras.layers.Input((), name='input/label')\n\n    x = backbone(image)\n    x = pooling(x)\n    x = dropout(x)\n    x = dense(x)\n    x = margin([x, label])\n    x = softmax(x)\n    return tf.keras.Model(\n        inputs=[image, label], outputs=x)\n\n\nclass DistributedModel:\n\n    def __init__(self,\n                 input_size,\n                 n_classes,\n                 batch_size,\n                 finetuned_weights,\n                 dense_units,\n                 dropout_rate,\n                 scale,\n                 margin,\n                 optimizer,\n                 strategy,\n                 mixed_precision,\n                 clip_grad):\n\n        self.model = create_model(\n            input_shape=input_size,\n            n_classes=n_classes,\n            dense_units=dense_units,\n            dropout_rate=dropout_rate,\n            scale=scale,\n            margin=margin,)\n\n        self.input_size = input_size\n        self.global_batch_size = batch_size * strategy.num_replicas_in_sync\n\n        if finetuned_weights:\n            self.model.load_weights(finetuned_weights)\n\n        self.mixed_precision = mixed_precision\n        self.optimizer = optimizer\n        self.strategy = strategy\n        self.clip_grad = clip_grad\n\n        # loss function\n        self.loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n            from_logits=False, reduction=tf.keras.losses.Reduction.NONE)\n\n        # metrics\n        self.mean_loss_train = tf.keras.metrics.SparseCategoricalCrossentropy(\n            from_logits=False)\n        self.mean_accuracy_train = tf.keras.metrics.SparseTopKCategoricalAccuracy(\n            k=5)\n\n        if self.optimizer and self.mixed_precision:\n            self.optimizer = \\\n                tf.keras.mixed_precision.experimental.LossScaleOptimizer(\n                    optimizer, loss_scale='dynamic')\n\n    def _compute_loss(self, labels, probs):\n        per_example_loss = self.loss_object(labels, probs)\n        return tf.nn.compute_average_loss(\n            per_example_loss, global_batch_size=self.global_batch_size)\n\n    def _backprop_loss(self, tape, loss, weights):\n        gradients = tape.gradient(loss, weights)\n        if self.mixed_precision:\n            gradients = self.optimizer.get_unscaled_gradients(gradients)\n        clipped, _ = tf.clip_by_global_norm(gradients, clip_norm=self.clip_grad)\n        self.optimizer.apply_gradients(zip(clipped, weights))\n\n    def _train_step(self, inputs):\n        with tf.GradientTape() as tape:\n            probs = self.model(inputs, training=True)\n            loss = self._compute_loss(inputs[1], probs)\n            if self.mixed_precision:\n                loss = self.optimizer.get_scaled_loss(loss)\n        self._backprop_loss(tape, loss, self.model.trainable_weights)\n        self.mean_loss_train.update_state(inputs[1], probs)\n        self.mean_accuracy_train.update_state(inputs[1], probs)\n        return loss\n    \n    def _predict_step(self, inputs):\n        probs = self.model(inputs, training=False)\n        return probs\n    \n    @tf.function\n    def _distributed_train_step(self, dist_inputs):\n        per_replica_loss = self.strategy.run(self._train_step, args=(dist_inputs,))\n        return self.strategy.reduce(\n            tf.distribute.ReduceOp.SUM, per_replica_loss, axis=None)\n    \n    @tf.function\n    def _distributed_predict_step(self, dist_inputs):\n        probs = self.strategy.run(self._predict_step, args=(dist_inputs,))\n        return probs\n    \n    def train(self, train_ds, epochs, save_path):\n        for epoch in range(epochs):\n            dist_train_ds = self.strategy.experimental_distribute_dataset(train_ds)\n            dist_train_ds = tqdm.tqdm(dist_train_ds)\n            for i, inputs in enumerate(dist_train_ds):\n                loss = self._distributed_train_step(inputs)\n                dist_train_ds.set_description(\n                    \"TRAIN: Loss {:.3f}, Accuracy {:.3f}\".format(\n                        self.mean_loss_train.result().numpy(),\n                        self.mean_accuracy_train.result().numpy()\n                    )\n                )\n            if save_path:\n                self.model.save_weights(save_path)\n\n            self.mean_loss_train.reset_states()\n            self.mean_accuracy_train.reset_states()\n    \n    def predict(self, test_ds):\n        dist_test_ds = self.strategy.experimental_distribute_dataset(test_ds)\n        dist_test_ds = tqdm.tqdm(dist_test_ds)\n        # initialize accumulators\n        predictions = np.zeros([0,], dtype='int32')\n        confidences = np.zeros([0,], dtype='float32')\n        for inputs in dist_test_ds:\n            probs_replicates = self._distributed_predict_step(inputs)\n            probs_replicates = self.strategy.experimental_local_results(probs_replicates)\n            for probs in probs_replicates:\n                m = tf.gather(tf.shape(probs), 0)\n                probs_argsort = tf.argsort(probs, direction='DESCENDING')\n                # obtain predictions\n                idx1 = tf.stack([tf.range(m), tf.zeros(m, dtype='int32')], axis=1)\n                preds = tf.gather_nd(probs_argsort, idx1)\n                # obtain confidences\n                idx2 = tf.stack([tf.range(m), preds], axis=1)\n                confs = tf.gather_nd(probs, idx2)\n                # add to accumulator\n                predictions = np.concatenate([predictions, preds], axis=0)\n                confidences = np.concatenate([confidences, confs], axis=0)\n        return predictions, confidences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# skip training and predicting on public data\nif submission_df.id[0] != '00084cdf8f600d00':\n\n    train_ds = create_dataset(\n        df=train_df,\n        training=True,\n        batch_size=config['batch_size'],\n        input_size=config['input_size'],\n    )\n\n    test_ds = create_dataset(\n        df=submission_df,\n        training=False,\n        batch_size=config['batch_size'],\n        input_size=config['input_size'],\n    )\n\n    with strategy.scope():\n\n        optimizer = tf.keras.optimizers.SGD(\n            config['learning_rate'], momentum=config['momentum'])\n        \n        dist_model = DistributedModel(\n            input_size=config['input_size'],\n            n_classes=config['n_classes'],\n            batch_size=config['batch_size'],\n            finetuned_weights=None,\n            dense_units=config['dense_units'],\n            dropout_rate=config['dropout_rate'],\n            scale=config['scale'],\n            margin=config['margin'],\n            optimizer=optimizer,\n            strategy=strategy,\n            mixed_precision=False,\n            clip_grad=config['clip_grad'])\n\n        dist_model.train(\n            train_ds=train_ds, \n            epochs=config['n_epochs'], \n            save_path=None)#'model.h5')\n\n        preds, confs = dist_model.predict(\n            test_ds=test_ds)\n\n    \n    for i, (pred, conf) in enumerate(zip(preds, confs)):\n        # if conf < 0.1:\n        #     submission_df.at[i, 'landmarks'] = ''\n        # else:\n        submission_df.at[i, 'landmarks'] = f'{mapping[pred]} {conf}'\n            \n    submission_df = submission_df.set_index('id')\n    submission_df = submission_df.drop('label', axis=1)\n    submission_df = submission_df.drop('prob', axis=1)\n    submission_df = submission_df.drop('path', axis=1)\n    submission_df.to_csv('submission.csv')\nelse:\n    submission_df = submission_df.set_index('id')\n    submission_df = submission_df.drop('label', axis=1)\n    submission_df = submission_df.drop('prob', axis=1)\n    submission_df = submission_df.drop('path', axis=1)\n    submission_df.to_csv('submission.csv')\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}