{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Numpy\nimport numpy as np\nfrom numpy import loadtxt\n#Pandas\nimport pandas as pd\n#Tensorflow\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.utils import Sequence\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom tensorflow.keras import layers\nfrom keras.preprocessing.image import array_to_img \nfrom keras.preprocessing.image import ImageDataGenerator\n#OS\nimport os\nfrom os import listdir\nfrom os.path import isfile, join\n#albumentations\nimport albumentations as A\nfrom albumentations import Compose, HorizontalFlip, RandomBrightnessContrast, ToFloat, RGBShift,VerticalFlip\nfrom albumentations import RandomGamma,GaussNoise,GaussianBlur, RandomSizedCrop, Rotate\n#Other\nimport cv2 \nimport matplotlib.pyplot as plt\nfrom sklearn import preprocessing\nfrom PIL import Image\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-12T09:33:49.057374Z","iopub.execute_input":"2022-05-12T09:33:49.057677Z","iopub.status.idle":"2022-05-12T09:33:49.066919Z","shell.execute_reply.started":"2022-05-12T09:33:49.057641Z","shell.execute_reply":"2022-05-12T09:33:49.066143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"../input/hotel-id-to-combat-human-trafficking-2022-fgvc9/\"\n\ntrain_dir = path + 'train_images/'\ntest_dir = path + 'test_images/'\nvalidation_dir = path + 'train_masks'\nIM_SIZE = 224\nBATCH_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2022-05-12T09:33:49.080861Z","iopub.execute_input":"2022-05-12T09:33:49.081238Z","iopub.status.idle":"2022-05-12T09:33:49.089097Z","shell.execute_reply.started":"2022-05-12T09:33:49.081209Z","shell.execute_reply":"2022-05-12T09:33:49.088166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Finds all training images and links them to their hotel number\nhotels = np.array([])\nimages = np.array([])\nfor hotel in tqdm(os.listdir(train_dir)):\n    for file in os.listdir(train_dir + hotel):\n        hotels = np.append(hotels, hotel)\n        images = np.append(images, file)\n        \n#Puts all hotels into a dataframe \nd = {'image': images, 'hotel_id': hotels}\ntrain = pd.DataFrame(data=d)     \n\n#And then encdoes the labels\nle = preprocessing.LabelEncoder()\nle.fit(train['hotel_id'])\ntrain['label'] = le.transform(train['hotel_id'])\n\n#And adds a couple of other variables\ntrain['chain_image'] = train['hotel_id'].astype(str) + '/' + train['image']\ntrain['augmented'] = False\ntrain[:5]","metadata":{"execution":{"iopub.status.busy":"2022-05-12T09:33:49.094222Z","iopub.execute_input":"2022-05-12T09:33:49.094479Z","iopub.status.idle":"2022-05-12T09:35:00.507920Z","shell.execute_reply.started":"2022-05-12T09:33:49.094451Z","shell.execute_reply":"2022-05-12T09:35:00.507064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Calculate the weights of each of the hotels to account for the difference in hotel occurances.\nhotel_weight = np.array([a for a in train.groupby(['label']).size()])\nhotel_weight = np.sum(hotel_weight)/(len(train['label'].unique()) * hotel_weight)\nhotel_weight = dict(enumerate(hotel_weight))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-12T09:35:00.509900Z","iopub.execute_input":"2022-05-12T09:35:00.510144Z","iopub.status.idle":"2022-05-12T09:35:00.525546Z","shell.execute_reply.started":"2022-05-12T09:35:00.510115Z","shell.execute_reply":"2022-05-12T09:35:00.524328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Augmentation of the images is done using the augmentor1 function\ndef augmentor1(imgArr):\n    sizes = np.min(imgArr.shape[0:2])\n    augmentation1 = A.Compose([\n                        HorizontalFlip(p = 0.5), \n                        Rotate(), #might be bad for the network??\n                        RandomBrightnessContrast(p = 1), \n                        RandomSizedCrop(min_max_height= [int(sizes*0.6),sizes], height= IM_SIZE, width = IM_SIZE, p = 0.5), GaussianBlur(p = 0.25),\n                        GaussNoise(p=0.25, var_limit=10/255), \n                        A.CoarseDropout(p=0.5, min_holes=1, max_holes=6, \n                                        min_height=IM_SIZE//16, max_height=IM_SIZE//4,\n                                        min_width=IM_SIZE//16,  max_width=IM_SIZE//4), # normal coarse dropout\n                        A.CoarseDropout(p=0.75, max_holes=1, \n                                        min_height=IM_SIZE//4, max_height=IM_SIZE//2,\n                                        min_width=IM_SIZE//4,  max_width=IM_SIZE//2, \n                                        fill_value=(255,0,0))\n                        ])\n    return augmentation1(image = imgArr/255)['image']*255\n\nimage1 = np.array(cv2.imread(train_dir+'10054/000039550.jpg'), dtype = 'float32')\nfig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(15,15))\nax1.imshow(image1/255)\nax2.imshow(augmentor1(image1)/255)\nax3.imshow(augmentor1(image1)/255)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T09:35:00.526971Z","iopub.execute_input":"2022-05-12T09:35:00.527256Z","iopub.status.idle":"2022-05-12T09:35:01.445535Z","shell.execute_reply.started":"2022-05-12T09:35:00.527200Z","shell.execute_reply":"2022-05-12T09:35:01.444179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Code used from https://stackoverflow.com/questions/51355524/use-multiple-directories-for-flow-from-directory-in-keras\n\nclass MergedGenerators(Sequence):\n\n    def __init__(self, batch_size, generators=[], sub_batch_size=[]):\n        self.generators = generators\n        self.sub_batch_size = sub_batch_size\n        self.batch_size = batch_size\n\n    def __len__(self):\n        return int(\n            sum([(len(self.generators[idx]) * self.sub_batch_size[idx])\n                 for idx in range(len(self.sub_batch_size))]) /\n            self.batch_size)\n\n    def __getitem__(self, index):\n        \"\"\"Getting items from the generators and packing them\"\"\"\n\n        X_batch = []\n        Y_batch = []\n        for generator in self.generators:\n            if generator.class_mode is None:\n                x1 = generator[index % len(generator)]\n                X_batch = [*X_batch, *x1]\n\n            else:\n                x1, y1 = generator[index % len(generator)]\n                X_batch = [*X_batch, *x1]\n                Y_batch = [*Y_batch, *y1]\n\n        if self.generators[0].class_mode is None:\n            return np.array(X_batch)\n        return tf.keras.applications.resnet50.preprocess_input(np.array(X_batch)), np.array(Y_batch)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T09:35:01.447943Z","iopub.execute_input":"2022-05-12T09:35:01.448216Z","iopub.status.idle":"2022-05-12T09:35:01.460115Z","shell.execute_reply.started":"2022-05-12T09:35:01.448178Z","shell.execute_reply":"2022-05-12T09:35:01.459082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def two_image_generator(generator, \n                        directory, \n                        batch_size,\n                        subset,\n                        shuffle = False,\n                        ): \n                        \n\n    gen1 = generator.flow_from_directory(\n        directory,\n        subset = subset,\n        target_size=(IM_SIZE, IM_SIZE),\n        batch_size=batch_size,\n        \n        class_mode='categorical',\n        shuffle = shuffle,\n        seed = 7)\n\n        \n\n    while True:\n        X1i = gen1.next()\n        yield [X1i[0], X1i[1]], X1i[1]\n        \n        \ntrain_image_generator = ImageDataGenerator(validation_split = 0.2, preprocessing_function = augmentor1)  \nvalidation_image_generator = ImageDataGenerator(validation_split = 0.2, preprocessing_function = None)\n\ntrain_generator = two_image_generator(train_image_generator, \n                                      train_dir,\n                                      batch_size = BATCH_SIZE,  \n                                      shuffle = True,\n                                      subset = 'training')\n\nvalidation_generator = two_image_generator(train_image_generator, \n                                      train_dir,\n                                      batch_size = BATCH_SIZE,  \n                                      shuffle = True, \n                                      subset = 'validation')","metadata":{"execution":{"iopub.status.busy":"2022-05-12T09:35:01.461412Z","iopub.execute_input":"2022-05-12T09:35:01.461713Z","iopub.status.idle":"2022-05-12T09:35:01.481087Z","shell.execute_reply.started":"2022-05-12T09:35:01.461681Z","shell.execute_reply":"2022-05-12T09:35:01.480325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#Creates the two data generators and merges them afterwards\ntrain_image_generator = ImageDataGenerator(validation_split = 0.2, preprocessing_function = augmentor1)\nvalidation_image_generator = ImageDataGenerator(validation_split = 0.2, preprocessing_function = None)\n\ntrain_images = data_generator1.flow_from_directory(train_dir,\n        target_size = (IM_SIZE, IM_SIZE),\n        subset=\"training\",\n        batch_size = BATCH_SIZE,\n        shuffle= False,\n        class_mode = 'categorical')\n\nvalidation_images = data_generator2.flow_from_directory(train_dir,\n        target_size = (IM_SIZE, IM_SIZE),\n        subset=\"validation\",\n        batch_size = BATCH_SIZE,\n        class_mode = 'categorical')\n\ntrain_labels = data_generator1.flow_from_directory(train_dir,\n        target_size = (IM_SIZE, IM_SIZE),\n        subset=\"training\",\n        batch_size = BATCH_SIZE,\n        shuffle= False,\n        class_mode = 'categorical')\n\n\ntrain_gen = MergedGenerators(\n        BATCH_SIZE,\n        generators=[ train_generator2],\n        sub_batch_size=[ BATCH_SIZE])\n\nval_gen = MergedGenerators(\n        BATCH_SIZE,\n        generators=[ validation_generator2],\n        sub_batch_size=[ BATCH_SIZE])","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:44:37.150025Z","iopub.execute_input":"2022-05-11T18:44:37.150342Z","iopub.status.idle":"2022-05-11T18:44:47.788487Z","shell.execute_reply.started":"2022-05-11T18:44:37.150307Z","shell.execute_reply":"2022-05-11T18:44:47.7876Z"}}},{"cell_type":"code","source":"#https://github.com/4uiiurz1/keras-arcface\nfrom keras import regularizers\nfrom keras import backend as K\n\nclass ArcFace(tf.keras.layers.Layer):\n    def __init__(self, n_classes=10, s=30.0, m=0.50, regularizer=None, **kwargs):\n        super(ArcFace, self).__init__(**kwargs)\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.regularizer = regularizers.get(regularizer)\n\n    def build(self, input_shape):\n        super(ArcFace, self).build(input_shape[0])\n        self.W = self.add_weight(name='W',\n                                shape=(input_shape[0][-1], self.n_classes),\n                                initializer='glorot_uniform',\n                                trainable=True,\n                                regularizer=self.regularizer)\n\n    def call(self, inputs):\n        x, y = inputs\n        c = K.shape(x)[-1]\n        # normalize feature\n        x = tf.nn.l2_normalize(x, axis=1)\n        # normalize weights\n        W = tf.nn.l2_normalize(self.W, axis=0)\n        # dot product\n        logits = x @ W\n        # add margin\n        # clip logits to prevent zero division when backward\n        theta = tf.acos(K.clip(logits, -1.0 + K.epsilon(), 1.0 - K.epsilon()))\n        target_logits = tf.cos(theta + self.m)\n        # sin = tf.sqrt(1 - logits**2)\n        # cos_m = tf.cos(logits)\n        # sin_m = tf.sin(logits)\n        # target_logits = logits * cos_m - sin * sin_m\n        #\n        logits = logits * (1 - y) + target_logits * y\n        # feature re-scale\n        logits *= self.s\n        out = tf.nn.softmax(logits)\n\n        return out\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.n_classes)\n    \n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'regularizer': self.regularizer,\n        })\n        return config\n","metadata":{"execution":{"iopub.status.busy":"2022-05-12T09:35:01.482148Z","iopub.execute_input":"2022-05-12T09:35:01.482869Z","iopub.status.idle":"2022-05-12T09:35:01.504225Z","shell.execute_reply.started":"2022-05-12T09:35:01.482833Z","shell.execute_reply":"2022-05-12T09:35:01.503768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#The neural network and its training step\ndef make_model(name):\n    inputs = tf.keras.layers.Input(shape=(IM_SIZE, IM_SIZE, 3))\n    labels = tf.keras.layers.Input(shape=(3116,))\n    #inputs = tf.keras.layers.Input((IM_SIZE, IM_SIZE, 3))\n    inputs = tf.keras.applications.resnet50.preprocess_input(inputs),\n    if name == 'ResNet50':\n        #pre_process = tf.keras.applications.resnet50.preprocess_input(inputs)\n        #model.add(pre_process)\n        \n        #model.add(tf.keras.applications.resnet50.preprocess_input())\n        \n        resnet_model = keras.applications.ResNet50(include_top = False,\n                                                weights='imagenet', \n                                                #classifier_activation=\"softmax\", \n                                                pooling = 'max',\n                                                #classes = 3116,\n                                                input_shape = (IM_SIZE, IM_SIZE, 3),\n                                                )\n        \n        for i in resnet_model.layers:\n            i.trainable = False\n        #resnet_model.summary()\n        embedding = resnet_model\n    \n    \n    if name == 'efficientnet_b0':\n        input_changed = tf.cast(inputs, tf.float32)\n        input_changed = tf.keras.applications.efficientnet.preprocess_input (input_changed)\n        embedding = keras.applications.EfficientNetB0(include_top = False,\n                                                weights='imagenet', \n                                                #classifier_activation=\"softmax\", \n                                                pooling = 'avg',\n                                                \n                                                )\n        embedding.trainable = False\n        \n    x = embedding(inputs)\n    x = tf.keras.layers.Flatten()(x)\n    outputs = ArcFace(n_classes=3116)([x, labels])\n    outputs = tf.keras.layers.SoftMax(outputs)\n    model = tf.keras.Model([inputs, labels], outputs)\n    #model.add(tf.keras.layers.Dense(3116, activation='Softmax'))\n    \n    #outputs = tf.keras.layers.Softmax()(outputs)\n    #model = tf.keras.Model(inputs, outputs)\n    callbacks = [\n        keras.callbacks.ModelCheckpoint(\"save_at_{epoch}.h5\"),\n    ]\n  \n    \n    model.compile(optimizer=tf.keras.optimizers.Adam(),\n                  loss='categorical_crossentropy', \n                  metrics=['accuracy', tf.metrics.Precision(top_k=5, name = 'MAP@5')],\n                  \n                 )\n    model.summary()\n  \n    return model, callbacks\n\n\n\nmodel, callbacks = make_model('ResNet50')\n\n","metadata":{"execution":{"iopub.status.busy":"2022-05-13T07:57:38.613101Z","iopub.execute_input":"2022-05-13T07:57:38.613705Z","iopub.status.idle":"2022-05-13T07:57:38.717088Z","shell.execute_reply.started":"2022-05-13T07:57:38.613608Z","shell.execute_reply":"2022-05-13T07:57:38.715339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = model.fit(train_generator,  epochs=1,\n          callbacks=tf.keras.callbacks.ModelCheckpoint('model.hdf5',\n                     verbose=1, save_best_only=True),\n          validation_data=validation_generator , \n          #class_weight = hotel_weight,)\n           # use_multiprocessing = True)\n          steps_per_epoch =50, \n          validation_steps = 10)","metadata":{"execution":{"iopub.status.busy":"2022-05-12T09:35:04.307300Z","iopub.execute_input":"2022-05-12T09:35:04.307551Z","iopub.status.idle":"2022-05-12T09:38:43.831270Z","shell.execute_reply.started":"2022-05-12T09:35:04.307513Z","shell.execute_reply":"2022-05-12T09:38:43.830382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#np.argwhere(train_gen[0][1][0]==1)#","metadata":{"execution":{"iopub.status.busy":"2022-05-12T09:38:43.832974Z","iopub.execute_input":"2022-05-12T09:38:43.833558Z","iopub.status.idle":"2022-05-12T09:38:43.837863Z","shell.execute_reply.started":"2022-05-12T09:38:43.833526Z","shell.execute_reply":"2022-05-12T09:38:43.836991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plt.imshow(train_gen[0][0][10]/255)\n#print(train_gen[0][1][10])205.63286","metadata":{"execution":{"iopub.status.busy":"2022-05-12T09:38:43.839664Z","iopub.execute_input":"2022-05-12T09:38:43.839969Z","iopub.status.idle":"2022-05-12T09:38:43.854911Z","shell.execute_reply.started":"2022-05-12T09:38:43.839946Z","shell.execute_reply":"2022-05-12T09:38:43.854084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-05T21:20:20.416137Z","iopub.execute_input":"2022-05-05T21:20:20.416834Z","iopub.status.idle":"2022-05-05T21:20:20.424552Z","shell.execute_reply.started":"2022-05-05T21:20:20.416794Z","shell.execute_reply":"2022-05-05T21:20:20.423872Z"}}}]}