{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Data Loading\n\nFirst step, we peek at the data paths:"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data.dataset import random_split     # spliting inTrain Val\nimport pandas as pd                                   # handling CSV\nimport os                                             # For File handling\nimport random                                         # Choosing from images dataset\nimport time                                           # timing Epochs  \nfrom tqdm.notebook import tqdm                        # Testing\nfrom os.path import join                              # File Handling\nfrom torchvision import transforms                    # Data Aug\nimport torch                                          # Framework\nfrom PIL import Image                                 # Loading Image\nfrom torch.utils.data import Dataset, DataLoader      # Dataset\nimport torch.nn.functional as F                       # Function\nimport json                                           # Loading Metadat\nfrom PIL import  ImageOps                             # Data Aug \nfrom PIL.Image import open as openIm                  # Image Handling\nimport matplotlib.pyplot  as plt                      # Ploting Image\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nstart_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we will load the sample submission and check."},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN       = \"../input/herbarium-2020-fgvc7/nybg2020/train/\"\nTEST        = \"../input/herbarium-2020-fgvc7/nybg2020/test/\"\nMETA        = \"metadata.json\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here is how the json file Looks Like\n\n"},{"metadata":{},"cell_type":"markdown","source":"TRAIN FILE"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(join(TRAIN,META),\"r\", encoding = \"ISO-8859-1\") as file:\n metadata = json.load(file)\nprint(\"Metadata has {} sections. These section has all the Information regarding Images in dataset like class, id, size etc. \".format(len(list(metadata.keys()))))\nprint(\"Let us see all the sections in metadata:- \", [print(\" - \",i) for i in list(metadata.keys())])\n\nprint(\"Number of Images in our Training set is:- \", len(metadata[\"images\"]))\nprint(\"\\n Let us see how every section of Dataset Looks like:-\\n\")\nfor i in list(metadata.keys()):\n print(\"- sample and number of elements in {} :- \".format(i),len(list(metadata[i])))\n print(\"\\t\",list(metadata[i])[0], end = \"\\n\\n\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"TEST FILE"},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(join(TEST,META),\"r\", encoding = \"ISO-8859-1\") as file:\n metadata_test = json.load(file)\nprint(\"Metadata has {} sections. These section has all the Information regarding Images in dataset like class, id, size etc. \".format(len(list(metadata_test.keys()))))\nprint(\"Let us see al the sections in metadata:- \", [print(\" - \",i) for i in list(metadata_test.keys())])\n\nprint(\"Number of Images in our Training set is:- \", len(metadata_test[\"images\"]))\nprint(\"\\n Let us see how every section of Dataset Looks like:-\\n\")\nfor i in list(metadata_test.keys()):\n print(\" - sample and number of elements in {} :- \".format(i),len(list(metadata_test[i])))\n print(\"\\t\",list(metadata_test[i])[0], end = \"\\n\\n\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are 1030747 Images in Train set.\n\nThere are 32094 Classes in The dataset."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"sample_sub = pd.read_csv('../input/herbarium-2020-fgvc7/sample_submission.csv')\ndisplay(sample_sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##Now, we will be unifying the metadata from the `*.json` files. We will first work with the `train` data."},{"metadata":{},"cell_type":"markdown","source":"Lets explore Train Data"},{"metadata":{},"cell_type":"markdown","source":"First, we access the `annotations` list and convert it to a df."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.DataFrame(metadata['annotations'])\ndisplay(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Next is for `plant categories`:"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_cat = pd.DataFrame(metadata['categories'])\ntrain_cat.columns = ['family', 'genus', 'category_id', 'category_name']\ndisplay(train_cat)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Followed by the `image properties`:"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img = pd.DataFrame(metadata['images'])\ntrain_img.columns = ['file_name', 'height', 'image_id', 'license', 'width']\ndisplay(train_img)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And lastly, the `region`:"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_reg = pd.DataFrame(metadata['regions'])\ntrain_reg.columns = ['region_id', 'region_name']\ndisplay(train_reg)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now let us see the Image Sample."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_img1 = pd.DataFrame(metadata['images'])\ntrain_ann1 = pd.DataFrame(metadata['annotations'])\ntrain_df1 = pd.merge(train_ann1, train_img1, left_on='image_id', right_on='id', how='left').drop('image_id', axis=1).sort_values(by=['category_id'])\ntrain_df1.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Then, we will merge all the DataFrames and see what we got:"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.merge(train_cat, on='category_id', how='outer')\ntrain_df = train_df.merge(train_img, on='image_id', how='outer')\ntrain_df = train_df.merge(train_reg, on='region_id', how='outer')\nprint(train_df.info())\ndisplay(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Lets see how images look like, printing one image here"},{"metadata":{"trusted":true},"cell_type":"code","source":"im = Image.open(\"../input/herbarium-2020-fgvc7/nybg2020/train/images/156/72/354106.jpg\")\nprint(\"Category Id is 15672 and Image Id is 354106 is shown below\")\nim","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now lets find out total unique values inn the final dataframe"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Total Unique Values for each columns:\")\nprint(\"{0:10s} \\t {1:10d}\".format('train_df', len(train_df)))\nfor col in train_df.columns:\n    print(\"{0:10s} \\t {1:10d}\".format(col, len(train_df[col].unique())))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This shows that we have images with different widht and height, lets resize the images to one scale"},{"metadata":{},"cell_type":"markdown","source":"The Distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"size_of_img = (28, 28)\nfig=plt.figure(figsize=(72,72))\nfor i in range(60):\n    ax=fig.add_subplot(12,12,i+1)\n    img = cv2.imread(TRAIN + metadata[\"images\"][i][\"file_name\"])\n    img = cv2.resize(img,size_of_img)\n    ax.imshow(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Lets find out and remove Nan values if any"},{"metadata":{"trusted":true},"cell_type":"code","source":"na = train_df.isnull().sum()\n#keep = [x for x in range(train_df.shape[0]) if not na[x]]\n#train_df = train_df.iloc[keep]\nna","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.dropna()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After selecting the `non-NaN` items, we now reiterate on their file types. We need to save on memory, as we reached `102+ MB` for this DataFrame Only."},{"metadata":{"trusted":true},"cell_type":"code","source":"#dtypes = ['int32', 'int32', 'int32', 'int32', 'object', 'object', 'object', 'object', 'int32', 'int32', 'int32', 'object']\n#for n, col in enumerate(train_df.columns):3\n #   train_df[col] = train_df[col].astype(dtypes[n])\n#print(train_df.info())\n#display(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Finally, for our `test` dataset. Since it only contains one key, `images`:"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame(metadata_test['images'])\ntest_df.columns = ['file_name', 'height', 'image_id', 'license', 'width']\nprint(test_df.info())\ndisplay(test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here, we can see that other than the `category_id`, there's also the `family`, `genus`, `category_name`, `region_id` and `region_name` for the other probable targets. `category_id` and `category_name` are one and the same, similar to `region_id` and `region_name`.\n\nA possible approach for this kernel is to use a `CNN` to predict `family` and `genus` (we will ignore `region` for now). Then, using the `family` and `genus`, we will predict the `category_id` for the image."},{"metadata":{"trusted":true},"cell_type":"code","source":"family = train_df[['family', 'genus', 'category_name']].groupby(['family', 'genus']).count()\ndisplay(family.describe())\ndisplay(family)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"With some proper `image_data_augmentation` we can make up for the small number of samples for some images (first quartile)."},{"metadata":{},"cell_type":"markdown","source":"# Model Creation\n\nIn this kernel, we will be creating a single model with multiple sparse-matrix outputs pertaining to: `family`, `genus`, and `category_id`.\n\nThe training should be: `family (trainable), genus (non-trainable), category_id(non-trainable)` until `family`'s accuracy reaches a certain level.\n\nAfter that, set `genus` to `trainable` until a certain limit, before setting `category_id` as `trainable`. This is due to their outputs linked to each other.\n\nHowever, due to it's complexity, I will just leave it alone for now... Probably going back to it in a future kernel."},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Dropout, Conv2D, MaxPool2D, Flatten, BatchNormalization, Input, concatenate\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.utils import plot_model\nfrom sklearn.model_selection import train_test_split as tts\n\nin_out_size = (120*120) + 3 #We will resize the image to 120*120 and we have 3 outputs\ndef xavier(shape, dtype=None):\n    return np.random.rand(*shape)*np.sqrt(1/in_out_size)\n\ndef fg_model(shape, lr=0.001):\n    '''Family-Genus model receives an image and outputs two integers indicating both the family and genus index.'''\n    i = Input(shape)\n    \n    x = Conv2D(3, (3, 3), activation='relu', padding='same', kernel_initializer=xavier)(i)\n    x = Conv2D(3, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(x)\n    x = MaxPool2D(pool_size=(3, 3), strides=(3,3))(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    x = Conv2D(16, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(x)\n    #x = Conv2D(16, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(x)\n    x = MaxPool2D(pool_size=(5, 5), strides=(5,5))(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    x = Flatten()(x)\n    \n    o1 = Dense(310, activation='softmax', name='family', kernel_initializer=xavier)(x)\n    \n    o2 = concatenate([o1, x])\n    o2 = Dense(3678, activation='softmax', name='genus', kernel_initializer=xavier)(o2)\n    \n    o3 = concatenate([o1, o2, x])\n    o3 = Dense(32094, activation='softmax', name='category_id', kernel_initializer=xavier)(o3)\n    \n    x = Model(inputs=i, outputs=[o1, o2, o3])\n    opt = Adam(lr=lr, amsgrad=True)\n    x.compile(optimizer=opt, loss=['sparse_categorical_crossentropy', \n                                   'sparse_categorical_crossentropy', \n                                   'sparse_categorical_crossentropy'],\n                 metrics=['accuracy'])\n    \n    return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Generator / Data Augmentation\n\nNow that we've designed our models, we will now proceed to our `data generator`."},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\ntrain_datagen = ImageDataGenerator(featurewise_center=False, featurewise_std_normalization=False, rotation_range=180, width_shift_range=0.1, height_shift_range=0.1, zoom_range=0.2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The data augmentation here is a little tricky, because we first want to crop away the pathological margins of the images, and then apply the data augmentation (rotation, shifts, zooms). This is particular important here, as we have very few data for some categories (for some, there are only ~5 images)."},{"metadata":{},"cell_type":"markdown","source":"Now, we will transform the `family` and `genus` to ids."},{"metadata":{"trusted":true},"cell_type":"code","source":"m = train_df[['file_name', 'family', 'genus', 'category_id']]\nfam = m.family.unique().tolist()\nm.family = m.family.map(lambda x: fam.index(x))\ngen = m.genus.unique().tolist()\nm.genus = m.genus.map(lambda x: gen.index(x))\ndisplay(m)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Kaggle Notebook Limit\n\nFor this dataset, I can't train the full dataset on this kernel as there's not enough RAM available. So, to bypass that, we will be training on a limited part of the dataset: a random sampled `50,000` items."},{"metadata":{},"cell_type":"markdown","source":"# Train\n\nNow, we will begin the training."},{"metadata":{"trusted":true},"cell_type":"code","source":"train, verif = tts(m, test_size=0.2, shuffle=True, random_state=17)\ntrain = train[:40000]\nverif = verif[:10000]\nshape = (120, 120, 3)\nepochs = 2\nbatch_size = 32\n\nmodel = fg_model(shape, 0.007)\n\n#Disable the last two output layers for training the Family\nfor layers in model.layers:\n    if layers.name == 'genus' or layers.name=='category_id':\n        layers.trainable = False\n\n#Train Family for 2 epochs\nmodel.fit_generator(train_datagen.flow_from_dataframe(dataframe=train,\n                                                      directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                                                      x_col=\"file_name\",\n                                                      y_col=[\"family\", \"genus\", \"category_id\"],\n                                                      target_size=(120, 120),\n                                                      batch_size=batch_size,\n                                                      class_mode='multi_output'),\n                    validation_data=train_datagen.flow_from_dataframe(\n                        dataframe=verif,\n                        directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                        x_col=\"file_name\",\n                        y_col=[\"family\", \"genus\", \"category_id\"],\n                        target_size=(120, 120),\n                        batch_size=batch_size,\n                        class_mode='multi_output'),\n                    epochs=epochs,\n                    steps_per_epoch=len(train)//batch_size,\n                    validation_steps=len(verif)//batch_size,\n                    verbose=1,\n                    workers=8,\n                    use_multiprocessing=False)\n\n#Reshuffle the inputs\ntrain, verif = tts(m, test_size=0.2, shuffle=True, random_state=17)\ntrain = train[:40000]\nverif = verif[:10000]\n\n#Make the Genus layer Trainable\nfor layers in model.layers:\n    if layers.name == 'genus':\n        layers.trainable = True\n        \n#Train Family and Genus for 2 epochs\nmodel.fit_generator(train_datagen.flow_from_dataframe(dataframe=train,\n                                                      directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                                                      x_col=\"file_name\",\n                                                      y_col=[\"family\", \"genus\", \"category_id\"],\n                                                      target_size=(120, 120),\n                                                      batch_size=batch_size,\n                                                      class_mode='multi_output'),\n                    validation_data=train_datagen.flow_from_dataframe(\n                        dataframe=verif,\n                        directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                        x_col=\"file_name\",\n                        y_col=[\"family\", \"genus\", \"category_id\"],\n                        target_size=(120, 120),\n                        batch_size=batch_size,\n                        class_mode='multi_output'),\n                    epochs=epochs,\n                    steps_per_epoch=len(train)//batch_size,\n                    validation_steps=len(verif)//batch_size,\n                    verbose=1,\n                    workers=8,\n                    use_multiprocessing=False)\n\n#Reshuffle the inputs\ntrain, verif = tts(m, test_size=0.2, shuffle=True, random_state=17)\ntrain = train[:40000]\nverif = verif[:10000]\n\n#Make the category_id layer Trainable\nfor layers in model.layers:\n    if layers.name == 'category_id':\n        layers.trainable = True\n        \n#Train them all for 2 epochs\nmodel.fit_generator(train_datagen.flow_from_dataframe(dataframe=train,\n                                                      directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                                                      x_col=\"file_name\",\n                                                      y_col=[\"family\", \"genus\", \"category_id\"],\n                                                      target_size=(120, 120),\n                                                      batch_size=batch_size,\n                                                      class_mode='multi_output'),\n                    validation_data=train_datagen.flow_from_dataframe(\n                        dataframe=verif,\n                        directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                        x_col=\"file_name\",\n                        y_col=[\"family\", \"genus\", \"category_id\"],\n                        target_size=(120, 120),\n                        batch_size=batch_size,\n                        class_mode='multi_output'),\n                    epochs=epochs,\n                    steps_per_epoch=len(train)//batch_size,\n                    validation_steps=len(verif)//batch_size,\n                    verbose=1,\n                    workers=8,\n                    use_multiprocessing=False)\nprint(model.history.keys())\n\n'''\nfor i in range(epochs):\n    n = 1\n    for X, Y in train_datagen.flow_from_dataframe(dataframe=train,\n                                                  directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                                                  x_col=\"file_name\",\n                                                  y_col=[\"family\", \"genus\", \"category_id\"],\n                                                  target_size=(120, 120),\n                                                  batch_size=batch_size,\n                                                  class_mode='multi_output'):\n        model.train_on_batch(X, Y, reset_metrics=False)\n        loss, fam_loss, gen_loss, cat_loss, fam_acc, gen_acc, cat_acc = model.evaluate(X, Y, verbose=False)\n        if n%10==0:\n            print(f\"For epoch {i} batch {n}: {loss}, {fam_loss}, {gen_loss}, {cat_loss}, {fam_acc}, {gen_acc}, {cat_acc}\")\n            for layers in model.layers:\n                if layers.name == 'family' and fam_acc>0.90:\n                    layers.trainable=False\n                elif layers.name == 'genus':\n                    if fam_acc>0.75:\n                        layers.trainable=True\n                    else:\n                        layers.trainable=False\n                elif layers.name == 'category_id':\n                    if fam_acc>0.75 and gen_acc>0.5:\n                        layers.trainable=True\n                    else:\n                        layers.trainable=False\n        n += 1\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('fg_model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Dropout, Conv2D, MaxPool2D, Flatten, BatchNormalization, Input, concatenate\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.utils import plot_model\nfrom sklearn.model_selection import train_test_split as tts\n\nin_out_size = (120*120) + 3 #We will resize the image to 120*120 and we have 3 outputs\ndef xavier(shape, dtype=None):\n    return np.random.rand(*shape)*np.sqrt(1/in_out_size)\n\ndef fg_model2(shape, lr=0.001):\n    '''Family-Genus model receives an image and outputs two integers indicating both the family and genus index.'''\n    i = Input(shape)\n    \n    x = Conv2D(3, (3, 3), activation='relu', padding='same', kernel_initializer=xavier)(i)\n    x = Conv2D(3, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(x)\n    x = MaxPool2D(pool_size=(3, 3), strides=(3,3))(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    x = Conv2D(8, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(i)\n    x = Conv2D(8, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(x)\n    x = MaxPool2D(pool_size=(5, 5), strides=(5,5))(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    x = Conv2D(16, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(x)\n    #x = Conv2D(16, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(x)\n    x = MaxPool2D(pool_size=(5, 5), strides=(5,5))(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    x = Flatten()(x)\n    \n    o1 = Dense(310, activation='softmax', name='family', kernel_initializer=xavier)(x)\n    \n    o2 = concatenate([o1, x])\n    o2 = Dense(3678, activation='softmax', name='genus', kernel_initializer=xavier)(o2)\n    \n    o3 = concatenate([o1, o2, x])\n    o3 = Dense(32094, activation='softmax', name='category_id', kernel_initializer=xavier)(o3)\n    \n    x = Model(inputs=i, outputs=[o1, o2, o3])\n    opt = Adam(lr=lr, amsgrad=True)\n    x.compile(optimizer=opt, loss=['sparse_categorical_crossentropy', \n                                   'sparse_categorical_crossentropy', \n                                   'sparse_categorical_crossentropy'],\n                 metrics=['accuracy'])\n    \n    return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, verif = tts(m, test_size=0.2, shuffle=True, random_state=17)\ntrain = train[:40000]\nverif = verif[:10000]\nshape = (120, 120, 3)\nepochs = 2\nbatch_size = 32\n\nmodel = fg_model2(shape, 0.007)\n\n#Disable the last two output layers for training the Family\nfor layers in model.layers:\n    if layers.name == 'genus' or layers.name=='category_id':\n        layers.trainable = False\n\n#Train Family for 2 epochs\nmodel.fit_generator(train_datagen.flow_from_dataframe(dataframe=train,\n                                                      directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                                                      x_col=\"file_name\",\n                                                      y_col=[\"family\", \"genus\", \"category_id\"],\n                                                      target_size=(120, 120),\n                                                      batch_size=batch_size,\n                                                      class_mode='multi_output'),\n                    validation_data=train_datagen.flow_from_dataframe(\n                        dataframe=verif,\n                        directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                        x_col=\"file_name\",\n                        y_col=[\"family\", \"genus\", \"category_id\"],\n                        target_size=(120, 120),\n                        batch_size=batch_size,\n                        class_mode='multi_output'),\n                    epochs=epochs,\n                    steps_per_epoch=len(train)//batch_size,\n                    validation_steps=len(verif)//batch_size,\n                    verbose=1,\n                    workers=8,\n                    use_multiprocessing=False)\n\n#Reshuffle the inputs\ntrain, verif = tts(m, test_size=0.2, shuffle=True, random_state=17)\ntrain = train[:40000]\nverif = verif[:10000]\n\n#Make the Genus layer Trainable\nfor layers in model.layers:\n    if layers.name == 'genus':\n        layers.trainable = True\n        \n#Train Family and Genus for 2 epochs\nmodel.fit_generator(train_datagen.flow_from_dataframe(dataframe=train,\n                                                      directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                                                      x_col=\"file_name\",\n                                                      y_col=[\"family\", \"genus\", \"category_id\"],\n                                                      target_size=(120, 120),\n                                                      batch_size=batch_size,\n                                                      class_mode='multi_output'),\n                    validation_data=train_datagen.flow_from_dataframe(\n                        dataframe=verif,\n                        directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                        x_col=\"file_name\",\n                        y_col=[\"family\", \"genus\", \"category_id\"],\n                        target_size=(120, 120),\n                        batch_size=batch_size,\n                        class_mode='multi_output'),\n                    epochs=epochs,\n                    steps_per_epoch=len(train)//batch_size,\n                    validation_steps=len(verif)//batch_size,\n                    verbose=1,\n                    workers=8,\n                    use_multiprocessing=False)\n\n#Reshuffle the inputs\ntrain, verif = tts(m, test_size=0.2, shuffle=True, random_state=17)\ntrain = train[:40000]\nverif = verif[:10000]\n\n#Make the category_id layer Trainable\nfor layers in model.layers:\n    if layers.name == 'category_id':\n        layers.trainable = True\n        \n#Train them all for 2 epochs\nmodel.fit_generator(train_datagen.flow_from_dataframe(dataframe=train,\n                                                      directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                                                      x_col=\"file_name\",\n                                                      y_col=[\"family\", \"genus\", \"category_id\"],\n                                                      target_size=(120, 120),\n                                                      batch_size=batch_size,\n                                                      class_mode='multi_output'),\n                    validation_data=train_datagen.flow_from_dataframe(\n                        dataframe=verif,\n                        directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                        x_col=\"file_name\",\n                        y_col=[\"family\", \"genus\", \"category_id\"],\n                        target_size=(120, 120),\n                        batch_size=batch_size,\n                        class_mode='multi_output'),\n                    epochs=epochs,\n                    steps_per_epoch=len(train)//batch_size,\n                    validation_steps=len(verif)//batch_size,\n                    verbose=1,\n                    workers=8,\n                    use_multiprocessing=False)\n#print(model.history.keys())\n\n'''\nfor i in range(epochs):\n    n = 1\n    for X, Y in train_datagen.flow_from_dataframe(dataframe=train,\n                                                  directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n                                                  x_col=\"file_name\",\n                                                  y_col=[\"family\", \"genus\", \"category_id\"],\n                                                  target_size=(120, 120),\n                                                  batch_size=batch_size,\n                                                  class_mode='multi_output'):\n        model.train_on_batch(X, Y, reset_metrics=False)\n        loss, fam_loss, gen_loss, cat_loss, fam_acc, gen_acc, cat_acc = model.evaluate(X, Y, verbose=False)\n        if n%10==0:\n            print(f\"For epoch {i} batch {n}: {loss}, {fam_loss}, {gen_loss}, {cat_loss}, {fam_acc}, {gen_acc}, {cat_acc}\")\n            for layers in model.layers:\n                if layers.name == 'family' and fam_acc>0.90:\n                    layers.trainable=False\n                elif layers.name == 'genus':\n                    if fam_acc>0.75:\n                        layers.trainable=True\n                    else:\n                        layers.trainable=False\n                elif layers.name == 'category_id':\n                    if fam_acc>0.75 and gen_acc>0.5:\n                        layers.trainable=True\n                    else:\n                        layers.trainable=False\n        n += 1\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict\n\nNow, we will do our prediction. We may as well skip doing a confusion-matrix for our model because it's not even fully trained, so we go straight to our submission.\n\nSimilar to the above reason, we will be limiting the `predictions` to the first `10,000` items due to RAM limitations."},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\ntest_datagen = ImageDataGenerator(featurewise_center=False,\n                                  featurewise_std_normalization=False)\n\ngenerator = test_datagen.flow_from_dataframe(\n        dataframe = test_df.iloc[:10000], #Limiting the test to the first 10,000 items\n        directory = '../input/herbarium-2020-fgvc7/nybg2020/test/',\n        x_col = 'file_name',\n        target_size=(120, 120),\n        batch_size=batch_size,\n        class_mode=None,  # only data, no labels\n        shuffle=False)\n\nfamily, genus, category = model.predict_generator(generator, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission\n\nNext, we'll save the predicted values under `predictions` into the specified format for submissions. Remember that our `predictions` is a `list` of 3-outputs, namely: `family`, `genus`, `category_id` in that order."},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nsub = pd.DataFrame()\nsub['Id'] = test_df.image_id\nsub['Id'] = sub['Id'].astype('int32')\nsub['Predicted'] = np.concatenate([np.argmax(category, axis=1), 23718*np.ones((len(test_df.image_id)-len(category)))], axis=0)\nsub['Predicted'] = sub['Predicted'].astype('int32')\ndisplay(sub)\nsub.to_csv('category_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Predicted'] = np.concatenate([np.argmax(family, axis=1), np.zeros((len(test_df.image_id)-len(family)))], axis=0)\nsub['Predicted'] = sub['Predicted'].astype('int32')\ndisplay(sub)\nsub.to_csv('family_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Predicted'] = np.concatenate([np.argmax(genus, axis=1), np.zeros((len(test_df.image_id)-len(genus)))], axis=0)\nsub['Predicted'] = sub['Predicted'].astype('int32')\ndisplay(sub)\nsub.to_csv('genus_submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Finish\n\nThere you have it! A working model for predicting the `Category` of the plants. I hope that this kernel helped you on your journey in unraveling the mysteries of this dataset! Please upvote before forking___________3-(^_^ )"},{"metadata":{"trusted":true},"cell_type":"code","source":"end_time = time.time()\ntotal = end_time - start_time\nh = total//3600\nm = (total%3600)//60\ns = total%60\nprint(\"Total time spent: %i hours, %i minutes, and %i seconds\" %(h, m, s))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}