{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-18T02:49:19.620103Z","iopub.execute_input":"2022-03-18T02:49:19.620826Z","iopub.status.idle":"2022-03-18T02:49:19.628952Z","shell.execute_reply.started":"2022-03-18T02:49:19.620758Z","shell.execute_reply":"2022-03-18T02:49:19.627877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# implement our own triplet loss so that we can handle multiples of same class","metadata":{}},{"cell_type":"markdown","source":"# Upvotes are appreciated!\n\n## I'd be happy to answer any questions I can in the comments! or talk off kaggle if you prefer so feel free to comment or reach out via the contact this user on kaggle\n### I am also open to any tips or corrections on my work! Constructive criticism is always appreciated.\n","metadata":{}},{"cell_type":"markdown","source":"# Notebook Overview\n* this notebook is a step two in this competition\n* the goal is to create a feature extractor for both whales and dolphins\n* this will help us pick out individuals in a species\n* we will use the google pretrained efficient nets with fine tuning to see if we can learn better","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"# DOWNLOADS\n# !pip install -q efficientnet\n!pip install tensorflow_addons\n!pip install wandb --upgrade\n\n# NORMAL IMPORTS\nimport os\nimport math\nfrom tqdm.notebook import tqdm, trange\nimport gc\nimport random\nimport pickle\nimport json\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport matplotlib\n%matplotlib inline\nimport matplotlib.pyplot as plt \nimport seaborn as sns\n\nimport sklearn\nprint(sklearn.__version__)\n\n# PREPROCESSING IMPORTS\nfrom sklearn.model_selection import StratifiedKFold, train_test_split, StratifiedShuffleSplit\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nfrom sklearn.preprocessing import OrdinalEncoder\n\n# MODEL BUILDING AND TRAINING IMPORTS\nimport tensorflow_hub as tfhub\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow_addons.losses import TripletSemiHardLoss,TripletHardLoss\n\n# EVALUATION IMPORTS\nfrom sklearn.neighbors import NearestNeighbors\n\n# LOGGING IMPORTS\nimport wandb\n\nBASE_PATH = '/kaggle/input/'\nprint(os.listdir(BASE_PATH))\nMAIN_DATASET = os.path.join(BASE_PATH, 'happy-whale-and-dolphin')\nCV_SPLITS_DATASET = os.path.join(BASE_PATH, 'happywhalefolds', 'folds.csv')\nCROPPED_DATASET = os.path.join(BASE_PATH, 'happywhale-cropped-dataset-yolov5-ds')\nOUTPUT_FOLDER = '/kaggle/working'","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:19.631135Z","iopub.execute_input":"2022-03-18T02:49:19.632141Z","iopub.status.idle":"2022-03-18T02:49:38.329936Z","shell.execute_reply.started":"2022-03-18T02:49:19.632084Z","shell.execute_reply":"2022-03-18T02:49:38.328965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:38.331495Z","iopub.execute_input":"2022-03-18T02:49:38.331798Z","iopub.status.idle":"2022-03-18T02:49:38.337178Z","shell.execute_reply.started":"2022-03-18T02:49:38.331767Z","shell.execute_reply":"2022-03-18T02:49:38.335797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The Setup\n* we want to setup the training strategy incase we use a tpu\n* we want to define the model config class that holds parameters\n* we want to seed the environment for reporducibility\n* link to our wandb project","metadata":{}},{"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n    \nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nAUTO = tf.data.experimental.AUTOTUNE\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:38.338941Z","iopub.execute_input":"2022-03-18T02:49:38.339177Z","iopub.status.idle":"2022-03-18T02:49:38.350727Z","shell.execute_reply.started":"2022-03-18T02:49:38.339149Z","shell.execute_reply":"2022-03-18T02:49:38.349752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config\n\n#### efficientnet is a python module we can use to download the pretrained efficient net model.\n\n#### The models were trained on a lot of image data, and according to the internet (and the fact a lot of other people are using them) they seem like a good option to extract information from our images\n\n#### We want to use a pretrained model that learned to extract meaningful features on a bunch of images because it is likely that it has learned some good general strategies. By starting with a model that generally preforms (or performs I dont know) well, we can leverage this feature extraction information on other images, and just fine tune the weights to learn to extract what we want from our images.","metadata":{}},{"cell_type":"code","source":"### These are for Efficient net version 2\nMODEL_TYPE = \"imagenet21k_b1\"\nMODEL_TASK = 'feature_vector'\nEFF_NET_PATH = \"https://tfhub.dev/google/imagenet/efficientnet_v2_{model_type}/{task}/2\".format(model_type=MODEL_TYPE, task=MODEL_TASK)\nprint(EFF_NET_PATH)\n\nWANDB_PROJECT = \"happy-whale\"\nWANDB_ENTITY = \"all-off-nothing\"\n\nBATCHSIZE = 64\n\nclass Config():\n    \"\"\"This class basically just holds information for us\n    \"\"\"\n    ### General\n    COMPETITION = \"Happy-Whale-And-Dolphin\"\n    AUTHOR = \"Kiernan\"\n    APPROACH = \"Vector-Embeddings\" # we could also do a classification or something between individuals\n    RUN_GROUP = wandb.util.generate_id()\n    SPECIES = 'dolphin'\n    \n    ### Hyper parameters\n    SEED = 123\n    BATCH_SIZE = BATCHSIZE * strategy.num_replicas_in_sync\n    EPOCHS = 100\n    \n    ### Model & Training Hyper Params\n    VECTOR_SIZE = 32\n    MARGIN = 1.0\n    SAMPLES_PER_CLASS = 8\n    LOSS = 'SEMIHARD'\n    ACTIVATION = 'relu'#'l2'\n    \n    ### LR Parameters\n    LR_STYLE = \"Ramp_Up_Then_Decay\"\n    LR_NOTES = \"Basic\"\n    \n    ### Where is the data from\n    DATA_ORIGIN = \"YOLOV5_Cropped_Dataset\"\n    DATA_METHOD = \"Bounding_Box_Cropping\"\n    FOLD_DATA = \"happywhalefolds\"\n    \n    ### Data params\n    IMAGE_SIZE = 240\n    IMAGE_CHANNELS = 3\n    ROTATION_FACTOR = 0.2\n    ZOOM_FACTOR=0.2\n    \n    ### Efficient net params\n    MODEL_GROUP = \"imagenet21k_b1\"\n    MODEL = f\"efficientnet_v2_{MODEL_TYPE}/{MODEL_TASK}/2\"\n    \n    \nconfig = Config()\n\nif(DEBUG):\n    config.EPOCHS = 2\n\nprint(config.MODEL)\nprint(f\"Batch size {config.BATCH_SIZE}\")","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:38.352475Z","iopub.execute_input":"2022-03-18T02:49:38.352797Z","iopub.status.idle":"2022-03-18T02:49:38.365852Z","shell.execute_reply.started":"2022-03-18T02:49:38.352755Z","shell.execute_reply":"2022-03-18T02:49:38.36502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wandb.keras import WandbCallback\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\napi_key = user_secrets.get_secret(\"WANDB\")\nwandb.login(key=api_key)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:38.367446Z","iopub.execute_input":"2022-03-18T02:49:38.367789Z","iopub.status.idle":"2022-03-18T02:49:39.012756Z","shell.execute_reply.started":"2022-03-18T02:49:38.367737Z","shell.execute_reply":"2022-03-18T02:49:39.011896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"We want to seed the env so that results are reproducible and more easily comparable.\n    \"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(config.SEED) ","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:39.014039Z","iopub.execute_input":"2022-03-18T02:49:39.014269Z","iopub.status.idle":"2022-03-18T02:49:39.132696Z","shell.execute_reply.started":"2022-03-18T02:49:39.014239Z","shell.execute_reply":"2022-03-18T02:49:39.131724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Setup\n* here we want to define the model helper functions\n* we want to create the model itself","metadata":{}},{"cell_type":"code","source":"### COPIED FROM THIS NOTEBOOK\n# https://www.kaggle.com/manojprabhaakr/effnet-b6-whale-comp\n\n\"\"\"\nI'm not well versed in why this is helpful, but aparently actually ramping up the learning rate at the start of training can perform very well.\n\"\"\"\n\ndef get_lr_callback(plot=False):\n    lr_start   = 0.00001\n    lr_max     = 0.00005 * config.BATCH_SIZE  \n    lr_min     = 0.000001\n    lr_ramp_ep = 4\n    lr_sus_ep  = 0\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n        return lr\n    if(plot):\n        epochs = list(range(config.EPOCHS))\n        learning_rates = [lrfn(x) for x in epochs]\n        plt.scatter(epochs,learning_rates)\n        ax = plt.gca()\n        ax.get_yaxis().get_major_formatter().set_scientific(False)\n        plt.show()\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\nget_lr_callback(plot=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:39.13757Z","iopub.execute_input":"2022-03-18T02:49:39.137802Z","iopub.status.idle":"2022-03-18T02:49:39.354219Z","shell.execute_reply.started":"2022-03-18T02:49:39.137775Z","shell.execute_reply":"2022-03-18T02:49:39.353211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nAgain I learned this on kaggle but apparently freezing the batch normalization layers from a pretrained model\ncan help prevent our model from overfitting. I learned this from other people in the competition\n\nTLDR from stack overflow\nthe new data we train on will likely have very different batch normalization parameters (on average the images are different from the ones the model was trained on)\nbatch norm is calculated over the train set after training is complete (during training it can be the mean and std of that batch or a rolling average).\nkeeping this frozen means that the ranges and values from the data will stay the same, requiring the model to keep the general pretrained learnings, and \nnot require it to relearn a ton of new information (in our case the parameter to data ratio is high so it could just memorize the training set)\nFrozen batch norm keeps the gradients smaller and less drastic so we can fine tune more without worrying about overfitting and losing the generalization capabilities\nwe wanted this pretrained model for to begin with\n(please correct me in comments if my understanding is bad, I'll update this section)\n\n\nhttps://www.kaggle.com/manojprabhaakr/effnet-b6-whale-comp\nhttps://stackoverflow.com/questions/63016740/why-its-necessary-to-frozen-all-inner-state-of-a-batch-normalization-layer-when#:~:text=This%20can%20help%20to%20minimize,in%20batch%20statistics%2C%20decreasing%20regularization.\nhttps://towardsdatascience.com/batch-norm-explained-visually-why-does-it-work-90b98bcc58a0\nhttps://towardsdatascience.com/batch-norm-explained-visually-how-it-works-and-why-neural-networks-need-it-b18919692739\n\"\"\"\n\ndef freeze_BN(model):\n    # Unfreeze layers while leaving BatchNorm layers frozen\n    for layer in model.layers:\n        if not isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.trainable = True\n        else:\n            layer.trainable = False","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:39.356043Z","iopub.execute_input":"2022-03-18T02:49:39.356505Z","iopub.status.idle":"2022-03-18T02:49:39.364665Z","shell.execute_reply.started":"2022-03-18T02:49:39.356458Z","shell.execute_reply":"2022-03-18T02:49:39.363713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def get_resizing_and_rescaling():\n#     with strategy.scope():\n#         resize_and_rescale = tf.keras.Sequential([\n#             tf.keras.layers.Resizing(config.IMAGE_SIZE, config.IMAGE_SIZE),\n#             tf.keras.layers.Rescaling(scale=1./127.5, offset=-1)\n#         ])\n#     return resize_and_rescale\n\n# resizer = get_resizing_and_rescaling()\n\ndef get_augmentation():\n    with strategy.scope():\n        augmentation = tf.keras.Sequential([\n            tf.keras.layers.RandomFlip(mode=\"horizontal\", seed=config.SEED),\n            tf.keras.layers.RandomZoom(config.ZOOM_FACTOR),\n            tf.keras.layers.RandomRotation(config.ROTATION_FACTOR)  \n        ])\n    return augmentation\n\naugmenter = get_augmentation()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:39.368761Z","iopub.execute_input":"2022-03-18T02:49:39.369246Z","iopub.status.idle":"2022-03-18T02:49:39.395324Z","shell.execute_reply.started":"2022-03-18T02:49:39.369202Z","shell.execute_reply":"2022-03-18T02:49:39.394466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def angular_distances(embeddings):\n    embeddings = tf.math.l2_normalize(embeddings, axis=-1)\n    angular_distances = 1 - tf.matmul(embeddings, tf.transpose(embeddings))\n    angular_distances = tf.maximum(angular_distances, 0.0)\n    mask_offdiag = tf.ones_like(angular_distances) - tf.linalg.diag(tf.ones([tf.shape(angular_distances)[0]]))\n    angular_distances = tf.math.multiply(angular_distances, mask_offdiag)\n    return angular_distances\n\ndef positive_angular(labels, embeddings):\n    adj = tf.equal(labels, tf.transpose(labels))\n    adj = tf.cast(adj, tf.float32) - tf.linalg.diag(tf.ones([tf.shape(labels)[0]]))\n    distances = angular_distances(embeddings)\n    pos_dist = tf.math.multiply(distances, adj)\n    pos_dist_mean = tf.reduce_mean(tf.ragged.boolean_mask(pos_dist, mask=tf.math.equal(adj, 1.0)))\n    return pos_dist_mean\n\ndef negative_angular(labels, embeddings):\n    adj = tf.math.logical_not(tf.equal(labels, tf.transpose(labels)))\n    adj = tf.cast(adj, tf.float32)\n    distances = angular_distances(embeddings)\n    neg_dist = tf.math.multiply(distances, adj)\n    neg_dist_mean = tf.reduce_mean(tf.ragged.boolean_mask(neg_dist, mask=tf.math.equal(adj, 1.0)))\n    return neg_dist_mean","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:39.396746Z","iopub.execute_input":"2022-03-18T02:49:39.396985Z","iopub.status.idle":"2022-03-18T02:49:39.409162Z","shell.execute_reply.started":"2022-03-18T02:49:39.396949Z","shell.execute_reply":"2022-03-18T02:49:39.40847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    \"\"\"This function returns the classification model. This inputs the images of whatever shape\n        we pick shape based on the expected image size of the different efficient net models\n        it then sends the images through efficient net and then uses a single node dense layer \n        with the sigmoid activation to binary classify an image as a whale or a dolphin\n    \"\"\"\n    tf.keras.backend.clear_session()\n    def internal_get_model():\n        # use the strategy scope becuase might be in tpu \n        # we want to make the model inside the scope of our strategy\n        # efficient net reads in an image of any size and spits out a 1000 dimension feature vector\n        with strategy.scope():\n            # input shape is (batch_size, image_size, image_size, image_channels)\n            inputs = tf.keras.layers.Input(shape=(None, None, config.IMAGE_CHANNELS), name=\"images\")\n\n#             x = resizer(inputs)\n            x = augmenter(inputs)\n\n            # input shape is (batch_size, any, any, 3)\n            x = tfhub.KerasLayer(EFF_NET_PATH, trainable=True)(x)\n\n            # takes the feature vector and turns it into a N dimensional vector\n            # l2 normalization means the euclidian distance of the vector will be 1\n            # kernel regularization provides loss based on the layers weights\n            # this promotes the model to maintain smaller weights\n            # this is important to stop overfitting as with this number of parameters to data the \n            # network could just memorize the data easily, this means the weights will be more evenly distributed\n            # and more likely to generalize\n            # l2 regilarization adds weight * sum of the squared coefficients\n            x = tf.keras.layers.Dropout(0.2)(x)\n            x = tf.keras.layers.Dense(config.VECTOR_SIZE)(x)\n            if(config.ACTIVATION == 'l2'):\n                outputs = tf.keras.layers.Lambda(lambda x: tf.math.l2_normalize(x, axis=-1))(x)\n            elif(config.ACTIVATION == 'relu'):\n                outputs = tf.keras.layers.Activation('relu')(x)\n            else:\n                raise Exception(f'Invalid activation values {config.ACTIVATION}')\n\n            # make the model\n            model = tf.keras.models.Model(inputs=inputs, outputs=outputs)\n\n            optimizer = tf.keras.optimizers.Adam()\n            if(config.LOSS == 'SEMIHARD'):\n                loss = TripletSemiHardLoss(config.MARGIN)\n            elif(config.LOSS == 'HARD'):\n                loss = TripletHardLoss(config.MARGIN)\n            else:\n                raise Excepttion(f'Invalid loss function specified {config.LOSS}')\n            model.compile(loss=loss, optimizer=optimizer, metrics=[positive_angular, negative_angular])\n        return model\n    model = internal_get_model()\n    freeze_BN(model)\n    return model\n\nmodel = get_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:39.410469Z","iopub.execute_input":"2022-03-18T02:49:39.411166Z","iopub.status.idle":"2022-03-18T02:49:46.901536Z","shell.execute_reply.started":"2022-03-18T02:49:39.411127Z","shell.execute_reply":"2022-03-18T02:49:46.900623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:46.90326Z","iopub.execute_input":"2022-03-18T02:49:46.903732Z","iopub.status.idle":"2022-03-18T02:49:49.84561Z","shell.execute_reply.started":"2022-03-18T02:49:46.903694Z","shell.execute_reply":"2022-03-18T02:49:49.844747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data\n* here we want to load the data","metadata":{}},{"cell_type":"code","source":"# load the main data file\nTEST_FOLD=0\ntrain_df = pd.read_csv(CV_SPLITS_DATASET)\ntrain_df = train_df.loc[train_df.encoded_species==config.SPECIES]\nif(DEBUG):\n    train_df = train_df.sample(frac=0.005)\nprint(train_df.shape)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:49.850309Z","iopub.execute_input":"2022-03-18T02:49:49.851025Z","iopub.status.idle":"2022-03-18T02:49:49.962732Z","shell.execute_reply.started":"2022-03-18T02:49:49.850979Z","shell.execute_reply":"2022-03-18T02:49:49.961838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(tf.keras.utils.Sequence):\n    def __init__(self, df, x_col='image', y_col='encoded_id'):\n        # save the meta info\n        self.base_dir = os.path.join(CROPPED_DATASET,'train_images', 'train_images')\n        self.x_col = x_col\n        self.y_col = y_col\n        \n        # image meta\n        self.dims = (config.IMAGE_SIZE, config.IMAGE_SIZE)\n        self.channels = 3\n        \n        # save the meta on what we will be choosing\n        self.batch_size = config.BATCH_SIZE\n        self.samples_per_class = config.SAMPLES_PER_CLASS\n        self.classes_per_batch = self.batch_size // self.samples_per_class\n        self.batches_per_epoch = (df.shape[0] // self.batch_size) + 1\n        \n        # save the conversions\n        self.labels_to_images = dict.fromkeys(df[y_col].unique(), [])\n        self.labels = df[y_col].unique()\n        for _, row in df.iterrows():\n            self.labels_to_images[row[y_col]].append(row[x_col])\n            \n        self.normalizer = tf.keras.layers.Rescaling(scale=1./127.5, offset=-1)\n        \n        super(CustomDataset, self).__init__()\n    \n    def __len__(self):\n        return self.batches_per_epoch\n\n    def on_epoch_end(self):\n        gc.collect()\n        return\n\n    def __getitem__(self, idx):\n        X = np.empty((self.batch_size, *self.dims, self.channels))\n        y = np.empty((self.batch_size), dtype=int)\n        individuals = np.random.choice(self.labels, self.classes_per_batch, replace=False)\n        batch_index = 0\n        for ind in individuals:\n            samples = np.random.choice(self.labels_to_images[ind], self.samples_per_class, replace=True)\n            for samp in samples:\n                X[batch_index, :, :, :] = tf.keras.utils.load_img(os.path.join(self.base_dir, samp), target_size=self.dims)\n                y[batch_index] = ind\n                batch_index += 1\n        return self.normalizer(X),y\n    \nclass TestCustomDataset(tf.keras.utils.Sequence):\n    def __init__(self, df, x_col='image', y_col='encoded_id'):\n        # save the meta info\n        self.base_dir = os.path.join(CROPPED_DATASET,'train_images', 'train_images')\n        self.x_col = x_col\n        self.y_col = y_col\n        \n        self.data = []\n        for _, row in df.iterrows():\n            self.data.append([row[self.x_col], row[self.y_col]])\n        self.data = np.array(self.data)\n        \n        # image meta\n        self.dims = (config.IMAGE_SIZE, config.IMAGE_SIZE)\n        self.channels = 3\n        \n        # save the meta on what we will be choosing\n        self.batch_size = config.BATCH_SIZE\n            \n        self.normalizer = tf.keras.layers.Rescaling(scale=1./127.5, offset=-1)\n        self.batches_per_epoch = math.ceil(self.data.shape[0] / self.batch_size)\n        self.labels = self.data[:, 1]\n        \n        super(TestCustomDataset, self).__init__()\n    \n    def __len__(self):\n        return self.batches_per_epoch\n\n    def on_epoch_end(self):\n        gc.collect()\n        return\n\n    def __getitem__(self, idx):\n        start = idx * self.batch_size\n        batch_size = self.batch_size\n        if(self.data.shape[0] < start + self.batch_size):\n            batch_size = self.data.shape[0] - start\n        X = np.empty((batch_size, *self.dims, self.channels))\n        y = np.empty((batch_size), dtype=int)\n        for i in range(batch_size):\n            X[i, :, :, :] = tf.keras.utils.load_img(os.path.join(self.base_dir, self.data[start+i, 0]), target_size=self.dims)\n            y[i] = self.data[start+i, 1]\n        return self.normalizer(X),y","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:49.964202Z","iopub.execute_input":"2022-03-18T02:49:49.964421Z","iopub.status.idle":"2022-03-18T02:49:49.990504Z","shell.execute_reply.started":"2022-03-18T02:49:49.964394Z","shell.execute_reply":"2022-03-18T02:49:49.989426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss Function","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the model","metadata":{}},{"cell_type":"code","source":"def train_loop(model, train_ds, val_ds, fold):\n    stopper = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, mode='min', restore_best_weights=True)\n    lr_scheduler = get_lr_callback()\n    hist = model.fit(train_ds,\n                     validation_data=val_ds,\n                     epochs=config.EPOCHS,\n                     callbacks=[stopper, lr_scheduler, WandbCallback()])\n    run.finish()\n    del stopper\n    del lr_scheduler\n    return hist\n        \ndef get_data(fold):\n    assert(TEST_FOLD < fold and fold in train_df.fold.unique())\n    fold_train_data = train_df.loc[np.logical_and(train_df.fold!=fold, train_df.fold!=TEST_FOLD)]\n    fold_val_data = train_df.loc[train_df.fold==fold]\n    print(fold_train_data.shape[0], fold_val_data.shape[0], train_df.loc[train_df.fold==TEST_FOLD].shape[0])\n    train_ds = CustomDataset(fold_train_data)\n    val_ds = CustomDataset(fold_val_data)\n    return train_ds, val_ds\n        \ndef train_model(fold):\n    train_ds, val_ds = get_data(fold)\n    model = get_model()\n    hist = train_loop(model, train_ds, val_ds, fold)\n    del train_ds\n    del val_ds\n    return model, hist\n\ndef test_model(model, fold, run):\n    test_ds = CustomDataset(train_df.loc[train_df.fold==TEST_FOLD])\n    ev = model.evaluate(test_ds)\n    run.log({'loss':ev})\n    del test_ds\n    return\n\n# def graph_training(hist):\n#     plt.figure(figsize=(8,8), tight_layout=True)\n#     plt.plot(hist.history['loss'])\n#     plt.plot(hist.history['val_loss'])\n#     plt.title('MSE Loss')\n#     plt.ylabel('Error')\n#     plt.xlabel('Epoch')\n#     plt.legend(['train', 'val'], loc='upper right')\n#     plt.show()\n\ndef train_on_fold(fold):\n    run = wandb.init(project=WANDB_PROJECT, entity=WANDB_ENTITY, group=config.RUN_GROUP, name=f\"{config.SPECIES}_fold{fold}_{config.RUN_GROUP}\", config=config.__dict__, job_type=\"train\")\n    print(f\"FOLD {fold}\")\n    print(\"===================================================\")\n    # open the run here\n    model, hist = train_model(fold, run)\n    test_model(model, fold, run)\n    model.save(os.path.join(OUTPUT_FOLDER, f\"Model_{fold}\"))\n    # close the run here \n    # custom test metrics on knn and on cosine\n    # similrity\n    run.finish()\n    del hist\n#     del model\n    plt.clf()\n    gc.collect()\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:49.992089Z","iopub.execute_input":"2022-03-18T02:49:49.992318Z","iopub.status.idle":"2022-03-18T02:49:50.010029Z","shell.execute_reply.started":"2022-03-18T02:49:49.992289Z","shell.execute_reply":"2022-03-18T02:49:50.009126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = train_on_fold(fold=1)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:49:50.011464Z","iopub.execute_input":"2022-03-18T02:49:50.011748Z","iopub.status.idle":"2022-03-18T02:51:31.843297Z","shell.execute_reply.started":"2022-03-18T02:49:50.011715Z","shell.execute_reply":"2022-03-18T02:51:31.842197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = train_on_fold(fold=2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = train_on_fold(fold=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = train_on_fold(fold=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(project=WANDB_PROJECT, entity=WANDB_ENTITY, group=config.RUN_GROUP, name=f\"{config.SPECIES}_test_fold{TEST_FOLD}_{config.RUN_GROUP}\", config=config.__dict__, job_type=\"eval\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_train_data = train_df.loc[train_df.fold!=TEST_FOLD]\nevaluate_test_data = train_df.loc[train_df.fold==TEST_FOLD]\nevaluate_train_ds = TestCustomDataset(evaluate_train_data)\nevaluate_test_ds = TestCustomDataset(evaluate_test_data)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:51:31.844723Z","iopub.execute_input":"2022-03-18T02:51:31.844981Z","iopub.status.idle":"2022-03-18T02:51:31.859026Z","shell.execute_reply.started":"2022-03-18T02:51:31.844948Z","shell.execute_reply":"2022-03-18T02:51:31.858133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NEIGHBORS = 1000\nif(DEBUG):\n    NEIGHBORS = 50","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:51:31.860434Z","iopub.execute_input":"2022-03-18T02:51:31.860993Z","iopub.status.idle":"2022-03-18T02:51:31.872502Z","shell.execute_reply.started":"2022-03-18T02:51:31.860947Z","shell.execute_reply":"2022-03-18T02:51:31.871906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_embeddings = model.predict(evaluate_train_ds)\nprint(train_embeddings.shape)\nneighbors = NearestNeighbors(n_neighbors=NEIGHBORS,metric='cosine')\nneighbors.fit(train_embeddings, evaluate_train_ds.labels)\nallowed_classes = set(evaluate_train_ds.labels.astype(np.int32))\n# del model\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:51:31.873494Z","iopub.execute_input":"2022-03-18T02:51:31.874062Z","iopub.status.idle":"2022-03-18T02:51:37.054191Z","shell.execute_reply.started":"2022-03-18T02:51:31.874029Z","shell.execute_reply":"2022-03-18T02:51:37.053481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NEW_INDIVIDUAL_ID=-1\ntest_embeddings = model.predict(evaluate_test_ds)\ntest_predictions = []\nfor test_embedding in test_embeddings:\n    distances, indexes = neighbors.kneighbors(test_embedding.reshape(1,-1), NEIGHBORS, return_distance=True)\n    predictions = list(dict.fromkeys(indexes[0,:]))\n    predictions = predictions[:4]\n    predictions.append(NEW_INDIVIDUAL_ID)\n    test_predictions.append(predictions)\ntest_predictions = np.array(test_predictions)","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:51:37.055411Z","iopub.execute_input":"2022-03-18T02:51:37.05577Z","iopub.status.idle":"2022-03-18T02:51:38.119678Z","shell.execute_reply.started":"2022-03-18T02:51:37.055739Z","shell.execute_reply":"2022-03-18T02:51:38.11894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_targets = evaluate_test_ds.labels.astype(np.int32).copy()\ntest_targets = np.array([x if x in allowed_classes else -1 for x in test_targets])\ntop_k_accuracy = 0\nfor target, predictions in zip(test_targets, test_predictions):\n    if(target in predictions):\n        top_k_accuracy += 1\naccuracy = top_k_accuracy / test_targets.shape[0]\nprint(f\"Top K Accuracy: \", )\nrun.log({'test/top_k_acc':accuracy})","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:51:38.120887Z","iopub.execute_input":"2022-03-18T02:51:38.121227Z","iopub.status.idle":"2022-03-18T02:51:38.12837Z","shell.execute_reply.started":"2022-03-18T02:51:38.121198Z","shell.execute_reply":"2022-03-18T02:51:38.127452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del evaluate_train_data\ndel evaluate_test_data\ndel evaluate_train_ds\ndel evaluate_test_ds\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-03-18T02:51:38.130115Z","iopub.execute_input":"2022-03-18T02:51:38.130482Z","iopub.status.idle":"2022-03-18T02:51:38.687291Z","shell.execute_reply.started":"2022-03-18T02:51:38.130438Z","shell.execute_reply":"2022-03-18T02:51:38.686197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run.finish()","metadata":{},"execution_count":null,"outputs":[]}]}