{"cells":[{"metadata":{},"cell_type":"markdown","source":"In the current competition there seem to be only a few kernels (ok .. besides the kernel provided by the competition organisers and related forks) that use a DL classification model.\n\nWith this kernel I hope to add a little to the choices that are available and still provide a nice score given the GPU time available. Some keypoints of this kernel:\n* Simple EfficientNet B2 model\n* Custom head\n* Custom sampling\n* An attempt to detect when an object is no landmark by using a Places365 model. I'am using the same model as in my notebook during last years Landmark Recognition [Landmark Notebook](https://www.kaggle.com/rsmits/keras-landmark-or-non-landmark-identification)\n\nIf you liked this kernel then please give it an upvote :-)\n\nI'am looking forward to any questions or remarks you might have that I can use to improve this kernel."},{"metadata":{},"cell_type":"markdown","source":"## Version Updates"},{"metadata":{},"cell_type":"markdown","source":"Version 10:\n* Increase image size to 256.\n* Slightly lower Batch Size and Learning Rate.\n* Lower the minimum sample threshold to include more classes.\n* Add Label Smoothing\n* Trained for 6 epochs on external M60 GPU VM.\n\nAltough I trained on an external GPU server the code to run the training is updated in the notebook. The score is however then in Version 8. I suspect this is due to the Label Smoothing. I will try the next version without it.. to be continued.."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import Modules\nimport albumentations as A\nimport cv2\nimport gc\nimport numpy as np\nimport os\nimport pandas as pd\nimport random\nimport sys\nfrom math import floor\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MultiLabelBinarizer, LabelBinarizer\n\n# EfficientNet\n!pip install '../input/glrec2020/Keras_Applications-1.0.8-py3-none-any.whl'\n!pip install '../input/glrec2020/efficientnet-1.1.0-py3-none-any.whl'\nimport efficientnet.tfkeras as efn\n\n# VGG 16 Places 365 scripts in custom dataset\nos.chdir(\"../input/keras-vgg16-places365\")\nfrom vgg16_places_365 import VGG16_Places365\nos.chdir(\"/kaggle/working/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set All Random Stuff\nSEED = 4249\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n\n#  Constants and Folders\nTEST_SIZE = 0.05\nSIZE = 256\nBATCH_SIZE = 24\nLR = 0.00035\nCHANNELS = 3\nSHAPE = (SIZE, SIZE, CHANNELS)\n\nTRAIN_ROOT_PATH = '../input/landmark-recognition-2020/train'\nTEST_ROOT_PATH = '../input/landmark-recognition-2020/test'\nsubmission = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\ntrain_data = pd.read_csv('../input/glrec2020/train.csv')\n\n# Perform Training or Inference (with or without Places Support)\nINFERENCE = True\nPLACES = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read Image\ndef read_image(path, im_size, normalize_image = False):\n    img = cv2.imread(path, cv2.IMREAD_COLOR)  \n    img = cv2.resize(img, (im_size, im_size))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32)\n    if normalize_image:\n        img /= 255.0  \n    return img","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([\n            A.HorizontalFlip(p = 0.2),\n            A.VerticalFlip(p = 0.2),\n            A.RandomRotate90(p = 0.2),\n            A.Transpose(p = 0.15),\n            A.ShiftScaleRotate(shift_limit = 0.05, scale_limit = 0.1, rotate_limit = 25, interpolation = 1, border_mode = 4, p = 0.15)\n        ], p = 0.9)\n\nclass TrainDataGenerator(tf.keras.utils.Sequence):\n    def __init__(self, image_ids, labels, classes, batch_size = 16, augmentation = False, *args, **kwargs):\n        self.image_ids = image_ids\n        self.labels = labels\n        self.classes = classes\n        self.batch_size = batch_size\n        self.augmentation = augmentation\n        self.indices = range(len(self.image_ids))\n        self.on_epoch_end()\n\n    def __len__(self):\n        return int(floor(len(self.image_ids) / self.batch_size))\n\n    def __getitem__(self, index):\n        indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]\n        X, Y = self.__data_generation(indices)\n        return X, Y\n\n    def on_epoch_end(self):\n        self.indices = np.arange(len(self.image_ids))\n    \n    def __data_generation(self, indices):\n        if self.augmentation:\n            augmentor = get_train_transforms()\n        X = np.empty((self.batch_size, *(SIZE, SIZE, CHANNELS)))\n        Y = np.empty((self.batch_size, self.classes))\n        \n        # Get Images for Batch\n        for i, index in enumerate(indices):\n            image_id, label = self.image_ids[index], self.labels[index]\n            image = read_image(f\"{TRAIN_ROOT_PATH}/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\", SIZE, normalize_image = True)\n            if self.augmentation:\n                data = {\"image\": image}\n                augmented = augmentor(**data)\n                image = augmented[\"image\"]\n            \n            X[i,] = image\n            Y[i,] = label.toarray()\n        \n        return X, Y ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train Dataset Default Summary\nprint(train_data.head())\nprint(train_data.shape)\n\n# Show default distribution of value_counts - ! Class Imbalance\ntrain_value_counts_regular = train_data['landmark_id'].value_counts()\nplt.figure(figsize=(12, 8))\nsns.distplot(train_value_counts_regular, hist = False, rug = False, label = \"Train Data normal value count distribution\")\nplt.xlabel(\"Value Counts\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Subsample Train.csv\ndef custom_sampler(df, min_samples, max_samples):\n    landmark_id_counts = df['landmark_id'].value_counts()\n\n    # First get part of data set between min_samples and max_samples\n    df1 = df[df['landmark_id'].isin(landmark_id_counts[(landmark_id_counts >= min_samples) & (landmark_id_counts <= max_samples)].index)]\n    \n    # Next for all Landmark_ids that have higher than max_samples sample count...down sample to max_samples\n    for id in landmark_id_counts[landmark_id_counts > max_samples].index:\n        # Get max_samples for specific landmark_id\n        temp_df = df[df.landmark_id == id].sample(max_samples, random_state = SEED)\n\n        # Concatenate Dataframes\n        df1 = pd.concat([df1, temp_df], axis = 0)\n        \n    return df1\n\n# Perform Custom datasampling on Train.csv\n# Only use Labels that have a minimum of min_samples in images.\n# Any Label that has more images than max_samples...downsample to the amount of max_samples\ntrain_data = custom_sampler(train_data, min_samples = 12, max_samples = 140)\n\n# New Distribution - Class Imbalance is smaller\ntrain_value_counts_custom = train_data['landmark_id'].value_counts()\nplt.figure(figsize=(12, 8))\nsns.distplot(train_value_counts_custom, hist = False, rug = False, label = \"Train Data custom value count distribution\")\nplt.xlabel(\"Value Counts\")\nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get all unique landmark_ids\nall_landmark_ids = train_data.landmark_id.unique().tolist()\nprint(f'Total number of classes sampled: {len(all_landmark_ids)}')\n\n# Get All Labels for One-Hot\nALL_LABELS = np.sort(np.unique(all_landmark_ids))\nlb = LabelBinarizer(sparse_output = True)\nlb.fit(ALL_LABELS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_gap(y_t, y_p):\n    pred_cat = tf.argmax(y_p, axis=-1)    \n    y_t_cat = tf.argmax(y_t, axis=-1) * tf.cast(\n        tf.reduce_sum(y_t, axis=-1), tf.int64)\n    \n    n_pred = tf.shape(pred_cat)[0]\n    is_c = tf.cast(tf.equal(pred_cat, y_t_cat), tf.float32)\n\n    GAP = tf.reduce_mean(\n          tf.cumsum(is_c) * is_c / tf.cast(\n              tf.range(1, n_pred + 1), \n              dtype=tf.float32))\n    \n    return GAP\n\n# Modelcheckpoint Callback\ndef ModelCheckpoint():\n    return tf.keras.callbacks.ModelCheckpoint(\n                            'Model_epoch{epoch:02d}_vl{val_loss:.4f}_va{val_acc:.4f}_vbg{val_batch_gap:.4f}.h5', \n                            monitor = 'val_loss', \n                            verbose = 1, \n                            save_best_only = False, \n                            save_weights_only = True, \n                            mode = 'min', \n                            save_freq = 'epoch')\n\n# Generalized mean pool - GeM\ndef generalized_mean_pool_2d(X):\n    gm_exp = 3.0\n    pool = (tf.reduce_mean(tf.abs(X**(gm_exp)), \n                        axis = [1, 2], \n                        keepdims = False) + 1.e-7)**(1./gm_exp)\n    return pool\n\n# I've also put in a model with a GeM layer. Given enough compute time a model with a GeM layer will likely score higher.\ndef create_model_gem(WEIGHTS, CLASSES): \n    # Input Layer\n    input = tf.keras.layers.Input(shape = SHAPE)\n    \n    # Create and Compile Model and show Summary\n    effnet_model = efn.EfficientNetB2(weights = WEIGHTS, include_top = False, input_tensor = input, pooling = None , classes = None)\n        \n    X = tf.keras.layers.Lambda(generalized_mean_pool_2d, name = 'gem')(effnet_model.output)\n    X = tf.keras.layers.Dropout(0.25)(X)\n    X = tf.keras.layers.Dense(1024, activation = 'relu')(X)\n    X = tf.keras.layers.BatchNormalization()(X)\n    X = tf.keras.layers.Dropout(0.25)(X)\n    preds = tf.keras.layers.Dense(CLASSES, activation = 'softmax')(X)\n    \n    # Create Final Model\n    model = tf.keras.Model(inputs = effnet_model.input, outputs = preds)\n\n    # UnFreeze all layers\n    for layer in model.layers:\n        layer.trainable = True\n    \n    return model\n\ndef create_model(WEIGHTS, CLASSES): \n    # Input Layer\n    input = tf.keras.layers.Input(shape = SHAPE)\n    \n    # Create and Compile Model and show Summary\n    effnet_model = efn.EfficientNetB2(weights = WEIGHTS, include_top = False, input_tensor = input, pooling = 'avg', classes = None)\n    \n    X = tf.keras.layers.Dropout(0.25)(effnet_model.output)\n    X = tf.keras.layers.Dense(1024, activation = 'relu')(X)\n    X = tf.keras.layers.BatchNormalization()(X)\n    X = tf.keras.layers.Dropout(0.25)(X)\n    preds = tf.keras.layers.Dense(CLASSES, activation = 'softmax')(X)\n    \n    # Create Final Model\n    model = tf.keras.Model(inputs = effnet_model.input, outputs = preds)\n\n    # UnFreeze all layers\n    for layer in model.layers:\n        layer.trainable = True\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not INFERENCE:\n    # Randomize Data\n    train_data = train_data.sample(frac = 1, random_state = SEED)\n\n    # Create Landmark Prediction Model\n    model = create_model('imagenet', len(all_landmark_ids))\n\n    # Compile Model\n    loss = tf.keras.losses.CategoricalCrossentropy(label_smoothing = 0.1)\n    optimizer = tf.keras.optimizers.Adam(lr = LR)\n    model.compile(loss = loss, optimizer = optimizer, metrics = ['acc', batch_gap])\n    \n    # Model Summary\n    #print(model.summary())\n\n    # Stratified Train Test Split\n    idsTrain, idsVal, labelsTrain, labelsVal = train_test_split(train_data.id.tolist(), \n                                                                lb.transform(train_data.landmark_id), \n                                                                test_size = TEST_SIZE, \n                                                                random_state = SEED, \n                                                                stratify = train_data.landmark_id)\n\n    # Create Data Generators for Train and Valid\n    data_generator_train = TrainDataGenerator(image_ids = idsTrain,\n                                            labels = labelsTrain, \n                                            classes = len(ALL_LABELS),\n                                            batch_size = BATCH_SIZE,\n                                            augmentation = True)\n    data_generator_val = TrainDataGenerator(image_ids = idsVal,\n                                            labels = labelsVal,\n                                            classes = len(ALL_LABELS),\n                                            batch_size = 2 * BATCH_SIZE,\n                                            augmentation = False)\n\n    TRAIN_STEPS = int(len(data_generator_train))\n    VALID_STEPS = int(len(data_generator_val))\n    print('Train Generator Size: {0}'.format(len(data_generator_train)))\n    print('Validation Generator Size: {0}'.format(len(data_generator_val)))\n\n    history = model.fit_generator(generator = data_generator_train,\n                        validation_data = data_generator_val,\n                        steps_per_epoch = TRAIN_STEPS,\n                        validation_steps = VALID_STEPS,\n                        epochs = 6,\n                        callbacks = [ModelCheckpoint()],\n                        verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"if INFERENCE:    \n    def create_image_tta(im_res):\n        # Flip Image Left-Right\n        im_res_lr = np.fliplr(im_res)\n        \n        return np.stack((im_res, im_res_lr))  \n\n    # Create Landmark Prediction Model\n    model = create_model(None, len(all_landmark_ids))\n\n    # Reload Weights\n    model.load_weights('../input/glrec2020/Model_epoch06_vl3.2417_va0.6774_vbg0.4803.h5')\n    \n    if PLACES:\n        # Places365 Model\n        model_places = VGG16_Places365(weights = '../input/keras-vgg16-places365/vgg16-places365_weights_tf_dim_ordering_tf_kernels.h5')\n        predictions_to_return = 6\n        places_classes = pd.read_csv('../input/keras-vgg16-places365/categories_places365_extended_v1.csv')\n        \n    print('Start Creating Submission...')\n    prediction = []\n    \n    for index, row in tqdm(submission.iterrows(), total = submission.shape[0]):\n        # Image Id\n        image_id = row['id']\n\n        # Get full file path\n        file_name = f\"{TEST_ROOT_PATH}/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\"\n\n        # Read and resize image\n        image = read_image(file_name, SIZE, normalize_image = True)\n\n        if PLACES:\n            # Read and resize image. For the Places Model I use a fixed size.\n            image_places = read_image(file_name, 224, normalize_image = False)\n\n            # Identify Indoor/Outdoor with Places365\n            places_preds = model_places.predict(create_image_tta(image_places))\n            places_pred = np.mean(places_preds, axis = 0)\n            places_top_preds = np.argsort(places_pred)[::-1][0:predictions_to_return]\n            \n            # Reset nonlandmark counter\n            counter = 0\n\n            # Get the Indoor/Outdoor Marker for the first 4 predictions. \n            # Update counter+1 if marker is NonLandmark      \n            if (places_classes.loc[places_classes['class'] == places_top_preds[0]].io == 1).bool():\n                counter +=1\n            if (places_classes.loc[places_classes['class'] == places_top_preds[1]].io == 1).bool():\n                counter +=1\n            if (places_classes.loc[places_classes['class'] == places_top_preds[2]].io == 1).bool():\n                counter +=1\n            if (places_classes.loc[places_classes['class'] == places_top_preds[3]].io == 1).bool():\n                counter +=1\n\n            # Get Predictions for Landmarks\n            if counter >= 3:\n                prediction.append(' ')                    \n            else:\n                pred = model.predict(create_image_tta(image))\n                max_value = np.max(np.mean(pred, axis = 0))\n                max_index = np.argmax(np.mean(pred, axis = 0))\n                prediction.append(str(ALL_LABELS[max_index]) + ' ' + str(max_value))\n        else:\n            pred = model.predict(create_image_tta(image))\n            max_value = np.max(np.mean(pred, axis = 0))\n            max_index = np.argmax(np.mean(pred, axis = 0))\n            prediction.append(str(ALL_LABELS[max_index]) + ' ' + str(max_value))\n            \n    # Create Submission CSV\n    submission['landmarks'] = np.array(prediction)\n    submission.to_csv('submission.csv', index = False)\n\n    # Show Summary\n    print(submission.head(25))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}