{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is a kernel forked from [Robin Smits](https://www.kaggle.com/rsmits) \nWell, I added some EDA before proceeding with the EffNet B2 model made by Robin.\n\n# Exploratory Data Analysis :"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nfrom scipy import stats\nimport glob\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/landmark-recognition-2020/train.csv\")\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check for Duplicates\ntrain.duplicated().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Training data size\",train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# missing data in training data \ntotal = train.isnull().sum().sort_values(ascending = False)\npercent = (train.isnull().sum()/train.isnull().count()).sort_values(ascending = False)\nmissing_train_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Occurance of landmark_id in decreasing order(Top categories)\ntemp = pd.DataFrame(train.landmark_id.value_counts().head(8))\ntemp.reset_index(inplace=True)\ntemp.columns = ['landmark_id','count']\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the most frequent landmark_ids\nplt.figure(figsize = (9, 8))\nplt.title('Most Frequent Landmarks')\nsns.set_color_codes(\"muted\")\nsns.barplot(x=\"landmark_id\", y=\"count\", data=temp, label=\"Count\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Occurance of landmark_id in increasing order\ntemp = pd.DataFrame(train.landmark_id.value_counts().tail(8))\ntemp.reset_index(inplace=True)\ntemp.columns = ['landmark_id','count']\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the least frequent landmark_ids\nplt.figure(figsize = (9, 8))\nplt.title('Least Frequent Landmarks')\nsns.set_color_codes(\"muted\")\nsns.barplot(x=\"landmark_id\", y=\"count\", data=temp, label=\"Count\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number of classes under 20 occurences\",(train['landmark_id'].value_counts() <= 20).sum(),'out of total number of categories',len(train['landmark_id'].unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Landmark id distribuition and density plot\nplt.figure(figsize = (10, 6))\nplt.title('Landmark ID Distribuition and Density Plot')\nsns.distplot(train['landmark_id'],color='tomato', kde=True,bins=250)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Line Plot\nsns.set()\nplt.figure(figsize = (10, 6))\nplt.title('Training Data: No. of Images per Class (Line Plot)')\nsns.set_color_codes(\"muted\")\nlandmarks_fold = pd.DataFrame(train['landmark_id'].value_counts())\nlandmarks_fold.reset_index(inplace=True)\nlandmarks_fold.columns = ['landmark_id','count']\nax = landmarks_fold['count'].plot(logy=True, grid=True)\nlocs, labels = plt.xticks()\nplt.setp(labels, rotation=30)\nax.set(xlabel=\"Landmarks\", ylabel=\"Number of Images\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Scatter Plot\n#sns.set()\n#plt.figure(figsize = (10, 6))\nsns.set_color_codes(\"muted\")\nlandmarks_fold_sorted = pd.DataFrame(train['landmark_id'].value_counts())\nlandmarks_fold_sorted.reset_index(inplace=True)\nlandmarks_fold_sorted.columns = ['landmark_id','count']\nlandmarks_fold_sorted = landmarks_fold_sorted.sort_values('landmark_id')\nsns.set(rc={'figure.figsize':(10.0,6.0)})\n#plt.title('Training Data: No. of Images per Class (Scatter Plot)')\nax = landmarks_fold_sorted.plot.scatter(x='landmark_id',y='count', title='Training Data: No. of Images per Class (Scatter Plot)', marker='o', color='b', edgecolor='k')\nlocs, labels = plt.xticks()\nplt.setp(labels, rotation=30)\nax.set(xlabel=\"Landmarks\", ylabel=\"Number of Images\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Having a Look at Some Images :"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_list = glob.glob('../input/landmark-recognition-2020/train/*/*/*/*')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams[\"axes.grid\"] = False\nf, axarr = plt.subplots(4, 3, figsize=(24, 22))\n\ncurr_row = 0\nfor i in range(12):\n    example = cv2.imread(train_list[i])\n    example = example[:,:,::-1]\n    \n    col = i%4\n    axarr[col, curr_row].imshow(example)\n    if col == 3:\n        curr_row += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TF Keras EffNet B2 Model :\n\n\n**Key Points from Robin's Work :**\n* Simple EfficientNet B2 model\n* Custom head\n* Custom sampling\n* An attempt to detect when an object is no landmark by using a Places365 model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import Modules\nimport albumentations as A\nimport cv2\nimport gc\nimport os\nimport random\nimport sys\nfrom math import floor\nfrom tqdm.notebook import tqdm\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MultiLabelBinarizer, LabelBinarizer\n\n# EfficientNet\n!pip install '../input/glrec2020/Keras_Applications-1.0.8-py3-none-any.whl'\n!pip install '../input/glrec2020/efficientnet-1.1.0-py3-none-any.whl'\nimport efficientnet.tfkeras as efn\n\n# VGG 16 Places 365 scripts in custom dataset\nos.chdir(\"../input/keras-vgg16-places365\")\nfrom vgg16_places_365 import VGG16_Places365\nos.chdir(\"/kaggle/working/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set All Random Stuff\nSEED = 12345\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)\n\n#  Constants and Folders\nTEST_SIZE = 0.25\nSIZE = 256\nBATCH_SIZE = 24\nLR = 0.0005\nCHANNELS = 3\nSHAPE = (SIZE, SIZE, CHANNELS)\n\nTRAIN_ROOT_PATH = '../input/landmark-recognition-2020/train'\nTEST_ROOT_PATH = '../input/landmark-recognition-2020/test'\nsubmission = pd.read_csv('../input/landmark-recognition-2020/sample_submission.csv')\ntrain_data = pd.read_csv('../input/glrec2020/train.csv')\n\n# Perform Training or Inference (with or without Places Support)\nINFERENCE = True\nPLACES = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read Image\ndef read_image(path, im_size, normalize_image = False):\n    img = cv2.imread(path, cv2.IMREAD_COLOR)  \n    img = cv2.resize(img, (im_size, im_size))\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32)\n    if normalize_image:\n        img /= 255.0  \n    return img","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([\n            A.HorizontalFlip(p = 0.2),\n            A.VerticalFlip(p = 0.2),\n            A.RandomRotate90(p = 0.2),\n            A.Transpose(p = 0.2),\n            A.ShiftScaleRotate(shift_limit = 0.05, scale_limit = 0.1, rotate_limit = 25, interpolation = 1, border_mode = 4, p = 0.15)\n        ], p = 0.8)\n\nclass TrainDataGenerator(tf.keras.utils.Sequence):\n    def __init__(self, image_ids, labels, classes, batch_size = 16, augmentation = False, *args, **kwargs):\n        self.image_ids = image_ids\n        self.labels = labels\n        self.classes = classes\n        self.batch_size = batch_size\n        self.augmentation = augmentation\n        self.indices = range(len(self.image_ids))\n        self.on_epoch_end()\n\n    def __len__(self):\n        return int(floor(len(self.image_ids) / self.batch_size))\n\n    def __getitem__(self, index):\n        indices = self.indices[index*self.batch_size:(index+1)*self.batch_size]\n        X, Y = self.__data_generation(indices)\n        return X, Y\n\n    def on_epoch_end(self):\n        self.indices = np.arange(len(self.image_ids))\n    \n    def __data_generation(self, indices):\n        if self.augmentation:\n            augmentor = get_train_transforms()\n        X = np.empty((self.batch_size, *(SIZE, SIZE, CHANNELS)))\n        Y = np.empty((self.batch_size, self.classes))\n        \n        # Get Images for Batch\n        for i, index in enumerate(indices):\n            image_id, label = self.image_ids[index], self.labels[index]\n            image = read_image(f\"{TRAIN_ROOT_PATH}/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\", SIZE, normalize_image = True)\n            if self.augmentation:\n                data = {\"image\": image}\n                augmented = augmentor(**data)\n                image = augmented[\"image\"]\n            \n            X[i,] = image\n            Y[i,] = label.toarray()\n        \n        return X, Y ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train Dataset Default Summary\nprint(train_data.head())\nprint(train_data.shape)\n\n# Show default distribution of value_counts - ! Class Imbalance\ntrain_value_counts_regular = train_data['landmark_id'].value_counts()\nplt.figure(figsize=(12, 8))\nsns.distplot(train_value_counts_regular, hist = False, rug = False, label = \"Train Data normal value count distribution\")\nplt.xlabel(\"Value Counts\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Subsample Train.csv\ndef custom_sampler(df, min_samples, max_samples):\n    landmark_id_counts = df['landmark_id'].value_counts()\n\n    # First get part of data set between min_samples and max_samples\n    df1 = df[df['landmark_id'].isin(landmark_id_counts[(landmark_id_counts >= min_samples) & (landmark_id_counts <= max_samples)].index)]\n    \n    # Next for all Landmark_ids that have higher than max_samples sample count...down sample to max_samples\n    for id in landmark_id_counts[landmark_id_counts > max_samples].index:\n        # Get max_samples for specific landmark_id\n        temp_df = df[df.landmark_id == id].sample(max_samples, random_state = SEED)\n\n        # Concatenate Dataframes\n        df1 = pd.concat([df1, temp_df], axis = 0)\n        \n    return df1\n\n# Perform Custom datasampling on Train.csv\n# Only use Labels that have a minimum of min_samples in images.\n# Any Label that has more images than max_samples...downsample to the amount of max_samples\ntrain_data = custom_sampler(train_data, min_samples = 12, max_samples = 140)\n\n# New Distribution - Class Imbalance is smaller\ntrain_value_counts_custom = train_data['landmark_id'].value_counts()\nplt.figure(figsize=(12, 8))\nsns.distplot(train_value_counts_custom, hist = False, rug = False, label = \"Train Data custom value count distribution\")\nplt.xlabel(\"Value Counts\")\nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get all unique landmark_ids\nall_landmark_ids = train_data.landmark_id.unique().tolist()\nprint(f'Total number of classes sampled: {len(all_landmark_ids)}')\n\n# Get All Labels for One-Hot\nALL_LABELS = np.sort(np.unique(all_landmark_ids))\nlb = LabelBinarizer(sparse_output = True)\nlb.fit(ALL_LABELS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_gap(y_t, y_p):\n    pred_cat = tf.argmax(y_p, axis=-1)    \n    y_t_cat = tf.argmax(y_t, axis=-1) * tf.cast(\n        tf.reduce_sum(y_t, axis=-1), tf.int64)\n    \n    n_pred = tf.shape(pred_cat)[0]\n    is_c = tf.cast(tf.equal(pred_cat, y_t_cat), tf.float32)\n\n    GAP = tf.reduce_mean(\n          tf.cumsum(is_c) * is_c / tf.cast(\n              tf.range(1, n_pred + 1), \n              dtype=tf.float32))\n    \n    return GAP\n\n# Modelcheckpoint Callback\ndef ModelCheckpoint():\n    return tf.keras.callbacks.ModelCheckpoint(\n                            'Model_epoch{epoch:02d}_vl{val_loss:.4f}_va{val_acc:.4f}_vbg{val_batch_gap:.4f}.h5', \n                            monitor = 'val_loss', \n                            verbose = 1, \n                            save_best_only = False, \n                            save_weights_only = True, \n                            mode = 'min', \n                            save_freq = 'epoch')\n\n# Generalized mean pool - GeM\ndef generalized_mean_pool_2d(X):\n    gm_exp = 3.0\n    pool = (tf.reduce_mean(tf.abs(X**(gm_exp)), \n                        axis = [1, 2], \n                        keepdims = False) + 1.e-7)**(1./gm_exp)\n    return pool\n\n# I've also put in a model with a GeM layer. Given enough compute time a model with a GeM layer will likely score higher.\ndef create_model_gem(WEIGHTS, CLASSES): \n    # Input Layer\n    input = tf.keras.layers.Input(shape = SHAPE)\n    \n    # Create and Compile Model and show Summary\n    effnet_model = efn.EfficientNetB2(weights = WEIGHTS, include_top = False, input_tensor = input, pooling = None , classes = None)\n        \n    X = tf.keras.layers.Lambda(generalized_mean_pool_2d, name = 'gem')(effnet_model.output)\n    X = tf.keras.layers.Dropout(0.10)(X)\n    X = tf.keras.layers.Dense(1024, activation = 'relu')(X)\n    X = tf.keras.layers.BatchNormalization()(X)\n    X = tf.keras.layers.Dropout(0.10)(X)\n    preds = tf.keras.layers.Dense(CLASSES, activation = 'softmax')(X)\n    \n    # Create Final Model\n    model = tf.keras.Model(inputs = effnet_model.input, outputs = preds)\n\n    # UnFreeze all layers\n    for layer in model.layers:\n        layer.trainable = True\n    \n    return model\n\ndef create_model(WEIGHTS, CLASSES): \n    # Input Layer\n    input = tf.keras.layers.Input(shape = SHAPE)\n    \n    # Create and Compile Model and show Summary\n    effnet_model = efn.EfficientNetB2(weights = WEIGHTS, include_top = False, input_tensor = input, pooling = 'avg', classes = None)\n    \n    X = tf.keras.layers.Dropout(0.1)(effnet_model.output)\n    X = tf.keras.layers.Dense(1024, activation = 'relu')(X)\n    X = tf.keras.layers.BatchNormalization()(X)\n    X = tf.keras.layers.Dropout(0.1)(X)\n    preds = tf.keras.layers.Dense(CLASSES, activation = 'softmax')(X)\n    \n    # Create Final Model\n    model = tf.keras.Model(inputs = effnet_model.input, outputs = preds)\n\n    # UnFreeze all layers\n    for layer in model.layers:\n        layer.trainable = True\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not INFERENCE:\n    # Randomize Data\n    train_data = train_data.sample(frac = 1, random_state = SEED)\n\n    # Create Landmark Prediction Model\n    model = create_model('imagenet', len(all_landmark_ids))\n\n    # Compile Model\n    loss = tf.keras.losses.CategoricalCrossentropy(label_smoothing = 0.25)\n    optimizer = tf.keras.optimizers.Adam(lr = LR)\n    model.compile(loss = loss, optimizer = optimizer, metrics = ['acc', batch_gap])\n    \n    # Model Summary\n    #print(model.summary())\n\n    # Stratified Train Test Split\n    idsTrain, idsVal, labelsTrain, labelsVal = train_test_split(train_data.id.tolist(), \n                                                                lb.transform(train_data.landmark_id), \n                                                                test_size = TEST_SIZE, \n                                                                random_state = SEED, \n                                                                stratify = train_data.landmark_id)\n\n    # Create Data Generators for Train and Valid\n    data_generator_train = TrainDataGenerator(image_ids = idsTrain,\n                                            labels = labelsTrain, \n                                            classes = len(ALL_LABELS),\n                                            batch_size = BATCH_SIZE,\n                                            augmentation = True)\n    data_generator_val = TrainDataGenerator(image_ids = idsVal,\n                                            labels = labelsVal,\n                                            classes = len(ALL_LABELS),\n                                            batch_size = 2 * BATCH_SIZE,\n                                            augmentation = False)\n\n    TRAIN_STEPS = int(len(data_generator_train))\n    VALID_STEPS = int(len(data_generator_val))\n    print('Train Generator Size: {0}'.format(len(data_generator_train)))\n    print('Validation Generator Size: {0}'.format(len(data_generator_val)))\n\n    history = model.fit_generator(generator = data_generator_train,\n                        validation_data = data_generator_val,\n                        steps_per_epoch = TRAIN_STEPS,\n                        validation_steps = VALID_STEPS,\n                        epochs = 6,\n                        callbacks = [ModelCheckpoint()],\n                        verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"if INFERENCE:    \n    def create_image_tta(im_res):\n        # Flip Image Left-Right\n        im_res_lr = np.fliplr(im_res)\n        \n        return np.stack((im_res, im_res_lr))  \n\n    # Create Landmark Prediction Model\n    model = create_model(None, len(all_landmark_ids))\n\n    # Reload Weights\n    model.load_weights('../input/glrec2020/Model_epoch06_vl3.2417_va0.6774_vbg0.4803.h5')\n    \n    if PLACES:\n        # Places365 Model\n        model_places = VGG16_Places365(weights = '../input/keras-vgg16-places365/vgg16-places365_weights_tf_dim_ordering_tf_kernels.h5')\n        predictions_to_return = 6\n        places_classes = pd.read_csv('../input/keras-vgg16-places365/categories_places365_extended_v1.csv')\n        \n    print('Start Creating Submission...')\n    prediction = []\n    \n    for index, row in tqdm(submission.iterrows(), total = submission.shape[0]):\n        # Image Id\n        image_id = row['id']\n\n        # Get full file path\n        file_name = f\"{TEST_ROOT_PATH}/{image_id[0]}/{image_id[1]}/{image_id[2]}/{image_id}.jpg\"\n\n        # Read and resize image\n        image = read_image(file_name, SIZE, normalize_image = True)\n\n        if PLACES:\n            # Read and resize image. For the Places Model I use a fixed size.\n            image_places = read_image(file_name, 224, normalize_image = False)\n\n            # Identify Indoor/Outdoor with Places365\n            places_preds = model_places.predict(create_image_tta(image_places))\n            places_pred = np.mean(places_preds, axis = 0)\n            places_top_preds = np.argsort(places_pred)[::-1][0:predictions_to_return]\n            \n            # Reset nonlandmark counter\n            counter = 0\n\n            # Get the Indoor/Outdoor Marker for the first 4 predictions. \n            # Update counter+1 if marker is NonLandmark      \n            if (places_classes.loc[places_classes['class'] == places_top_preds[0]].io == 1).bool():\n                counter +=1\n            if (places_classes.loc[places_classes['class'] == places_top_preds[1]].io == 1).bool():\n                counter +=1\n            if (places_classes.loc[places_classes['class'] == places_top_preds[2]].io == 1).bool():\n                counter +=1\n            if (places_classes.loc[places_classes['class'] == places_top_preds[3]].io == 1).bool():\n                counter +=1\n\n            # Get Predictions for Landmarks\n            if counter >= 3:\n                prediction.append(' ')                    \n            else:\n                pred = model.predict(create_image_tta(image))\n                max_value = np.max(np.mean(pred, axis = 0))\n                max_index = np.argmax(np.mean(pred, axis = 0))\n                prediction.append(str(ALL_LABELS[max_index]) + ' ' + str(max_value))\n        else:\n            pred = model.predict(create_image_tta(image))\n            max_value = np.max(np.mean(pred, axis = 0))\n            max_index = np.argmax(np.mean(pred, axis = 0))\n            prediction.append(str(ALL_LABELS[max_index]) + ' ' + str(max_value))\n            \n    # Create Submission CSV\n    submission['landmarks'] = np.array(prediction)\n    submission.to_csv('submission.csv', index = False)\n\n    # Show Summary\n    print(submission.head(25))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}