{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\n\nimport sklearn\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nimport tensorflow.keras as keras\n\nimport multiprocessing\n\nfrom tensorflow.keras.layers import Dense, Input, Conv2D\nfrom tensorflow.keras.applications import EfficientNetB0\n\nfrom kaggle_secrets import UserSecretsClient\n\n# from skimage.io import imread\nimport cv2\n\nfrom skimage.transform import resize\nimport numpy as np\nimport math\n\nimport wandb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"wandb_api\")\nwandb_user = user_secrets.get_secret(\"wandb_user\")\n\nwandb.login(key = wandb_api)\ninit = wandb.init(project = 'hotel-id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GLOBAL_SEED = 42\n\nnp.random.seed(GLOBAL_SEED)\ntf.random.set_seed(GLOBAL_SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_cores = multiprocessing.cpu_count()\nprint(f\"CPU Cores: {num_cores}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/hotel-id-2021-fgvc8/train.csv\")\n\n# train.image = train.image.astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle_path = \"../input/hotel-id-2021-fgvc8/train_images/\"\ntrain['full_filepath'] = kaggle_path + train.chain.astype(str) +\"/\"+ train.image.astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.iloc[0,4]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Subsample"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[train.chain.isin([0,1,2])]\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_subsample = 5000\ntrain = train.sample(n_subsample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_val, = train_test_split(train, test_size = 0.30,\n    stratify = train['chain'], random_state = GLOBAL_SEED, shuffle = True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_train.shape)\nprint(X_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_classes = X_train.chain.nunique()\n\nBATCH_SIZE = 64\nSTEPS_PER_EPOCH = len(X_train) // BATCH_SIZE\nEPOCHS = 50\n\nIMG_HEIGHT = 224\nIMG_WIDTH = 224\nIMG_SIZE = (IMG_HEIGHT, IMG_WIDTH)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TF Sequence Class - Faster Approach"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Based on https://www.tensorflow.org/api_docs/python/tf/keras/utils/Sequence\n# https://github.com/keras-team/keras/issues/12847\n# https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly\n# https://keunwoochoi.wordpress.com/2017/08/24/tip-fit_generator-in-keras-how-to-parallelise-correctly/\n\nclass HotelBatchSequence(tf.keras.utils.Sequence):\n    \n    def __init__(self, x_set, y_set, batch_size,\n                 img_size = (224, 224),\n                 augment = False):\n        \"\"\"\n        `x_set` is list of paths to the images\n        `y_set` are the associated classes.\n\n        \"\"\"\n        \n        self.x = x_set\n        self.y = y_set\n        self.batch_size = batch_size\n        self.img_size = img_size\n    \n    def __len__(self):\n        \"\"\"Denotes the number of batches per epoch\"\"\"\n        return math.ceil(len(self.x) / self.batch_size)\n    \n    def __getitem__(self, idx):\n        \"\"\"Generate one batch of data\"\"\"\n        \n        first_id = idx * self.batch_size\n        last_id =  (idx + 1) * (self.batch_size)\n        \n        batch_x = self.x[first_id:last_id]\n        batch_y = self.y[first_id:last_id]\n        \n        #Xs = np.array([resize(imread(file_name), self.img_size)\n        #      for file_name in batch_x])\n        # \n        #ys = np.array(batch_y)\n        \n        output = np.array([\n            resize(cv2.imread(file_name), self.img_size)\n                   for file_name in batch_x]), np.array(batch_y)\n        \n        return output\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TrainGenerator = HotelBatchSequence(X_train.full_filepath, \n                                    tf.keras.utils.to_categorical(X_train.chain),\n                                    BATCH_SIZE)\n\nValidGenerator = HotelBatchSequence(X_val.full_filepath, \n                                   tf.keras.utils.to_categorical(X_val.chain),\n                                   BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"efficientnet = EfficientNetB0(include_top=True, \n                              weights=None, \n                              input_shape = (IMG_HEIGHT, IMG_WIDTH, 3),\n                              classes = n_classes\n)\n\n# efficientnet.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = efficientnet\n\nmodel.compile(optimizer = 'adam',\n              loss = 'categorical_crossentropy',\n              metrics = 'accuracy')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Source: https://gist.github.com/Callidior/747eb767862c9d48f9d900a6373b16d1\n# Author: Callidior\n\n# Also: https://gist.github.com/jeremyjordan/5a222e04bb78c242f5763ad40626c452\n\nclass SGDR(tf.keras.callbacks.Callback):\n    \"\"\"\n    \n    # Source: https://gist.github.com/Callidior/747eb767862c9d48f9d900a6373b16d1\n    # Author: Callidior\n\n    This callback implements the learning rate schedule for\n    Stochastic Gradient Descent with warm Restarts (SGDR),\n    as proposed by Loshchilov & Hutter (https://arxiv.org/abs/1608.03983).\n    \n    The learning rate at each epoch is computed as:\n    lr(i) = min_lr + 0.5 * (max_lr - min_lr) * (1 + cos(pi * i/num_epochs))\n    \n    Here, num_epochs is the number of epochs in the current cycle, which starts\n    with base_epochs initially and is multiplied by mul_epochs after each cycle.\n    \n    # Example\n        ```python\n            sgdr = CyclicLR(min_lr=0.0, max_lr=0.05,\n                                base_epochs=10, mul_epochs=2)\n            model.compile(optimizer=keras.optimizers.SGD(decay=1e-4, momentum=0.9),\n                          loss=loss)\n            model.fit(X_train, Y_train, callbacks=[sgdr])\n        ```\n    \n    # Arguments\n        min_lr: minimum learning rate reached at the end of each cycle.\n        max_lr: maximum learning rate used at the beginning of each cycle.\n        base_epochs: number of epochs in the first cycle.\n        mul_epochs: factor with which the number of epochs is multiplied\n                after each cycle.\n    \"\"\"\n\n    def __init__(self, min_lr=0.0, max_lr=0.05, base_epochs=10, mul_epochs=2):\n        super(SGDR, self).__init__()\n\n        self.min_lr = min_lr\n        self.max_lr = max_lr\n        self.base_epochs = base_epochs\n        self.mul_epochs = mul_epochs\n\n        self.cycles = 0.\n        self.cycle_iterations = 0.\n        self.trn_iterations = 0.\n\n        self._reset()\n\n    def _reset(self, new_min_lr=None, new_max_lr=None,\n               new_base_epochs=None, new_mul_epochs=None):\n        \"\"\"Resets cycle iterations.\"\"\"\n        \n        if new_min_lr != None:\n            self.min_lr = new_min_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_base_epochs != None:\n            self.base_epochs = new_base_epochs\n        if new_mul_epochs != None:\n            self.mul_epochs = new_mul_epochs\n        self.cycles = 0.\n        self.cycle_iterations = 0.\n        \n    def sgdr(self):\n        \n        cycle_epochs = self.base_epochs * (self.mul_epochs ** self.cycles)\n        return self.min_lr + 0.5 * (self.max_lr - self.min_lr) * (1 + np.cos(np.pi * (self.cycle_iterations + 1) / cycle_epochs))\n        \n    def on_train_begin(self, logs=None):\n        \n        if self.cycle_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.max_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.sgdr())\n            \n    def on_epoch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        logs['lr'] = K.get_value(self.model.optimizer.lr)\n        \n        self.trn_iterations += 1\n        self.cycle_iterations += 1\n        if self.cycle_iterations >= self.base_epochs * (self.mul_epochs ** self.cycles):\n            self.cycles += 1\n            self.cycle_iterations = 0\n            K.set_value(self.model.optimizer.lr, self.max_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.sgdr())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# wandb_callback = wandb.keras.WandbCallback(log_weights=True)\nmodel_checkpoint = tf.keras.callbacks.ModelCheckpoint(\"tfmodels/weights.{epoch:02d}-{val_loss:.2f}.hdf5\")\ncosine_annealing_lr = SGDR(min_lr=0.0, max_lr=0.05, base_epochs=10, mul_epochs=2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(TrainGenerator,\n                    steps_per_epoch = STEPS_PER_EPOCH,\n                    validation_data = ValidGenerator,\n                    workers = num_cores,\n                    epochs = 2,\n                    use_multiprocessing = False,\n                    max_queue_size = 10,\n                    callbacks=[\n#                        wandb_callback, \n                         model_checkpoint,\n#                         cosine_annealing_lr\n                    ])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}