{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":18627,"databundleVersionId":1017327,"sourceType":"competition"}],"dockerImageVersionId":29862,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # Array, Linear Algebra\nfrom torch.utils.data.dataset import random_split # spliting inTrain Val\nimport pandas as pd # handling CSV\nimport os # For File handling\nimport random # Choosing from images dataset\nimport time # timing Epochs \nfrom tqdm.notebook import tqdm # Testing\nfrom os.path import join # File Handling\nfrom torchvision import transforms # Data Aug\nimport torch # Framework\nfrom PIL import Image # Loading Image\nfrom torch.utils.data import Dataset, DataLoader # Dataset\nimport torch.nn.functional as F # Function\nimport json # Loading Metadat\nfrom PIL import ImageOps # Data Aug \nfrom PIL.Image import open as openIm # Image Handling\nimport matplotlib.pyplot as plt # Ploting Image\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2024-02-20T10:58:04.723870Z","iopub.execute_input":"2024-02-20T10:58:04.724274Z","iopub.status.idle":"2024-02-20T10:58:04.732188Z","shell.execute_reply.started":"2024-02-20T10:58:04.724211Z","shell.execute_reply":"2024-02-20T10:58:04.731256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN = \"../input/herbarium-2020-fgvc7/nybg2020/train/\"\nTEST = \"../input/herbarium-2020-fgvc7/nybg2020/test/\"\nMETA = \"metadata.json\"\nBATCH_SIZE = 7\nNUM_WORKERS = 2\nBATCH_EVAL = 1\nSHUFFLE = True\nEPOCHS = 3\nRESIZE = (800, 600)\nCLASSES = 32094\nLENGTH = 2*CLASSES","metadata":{"execution":{"iopub.status.busy":"2024-02-20T10:58:08.397855Z","iopub.execute_input":"2024-02-20T10:58:08.398229Z","iopub.status.idle":"2024-02-20T10:58:08.403807Z","shell.execute_reply.started":"2024-02-20T10:58:08.398183Z","shell.execute_reply":"2024-02-20T10:58:08.402834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(join(TRAIN,META),\"r\", encoding = \"ISO-8859-1\") as file:\n metadata = json.load(file)\nprint(\"Metadata has {} sections. These section has all the Information regarding Images in dataset like class, id, size etc. \".format(len(list(metadata.keys()))))\nprint(\"Let us see al the sections in metadata:- \", [print(\" - \",i) for i in list(metadata.keys())])\n\nprint(\"Number of Images in our Training set is:- \", len(metadata[\"images\"]))\nprint(\"\\n Let us see how every section of Dataset Looks like:-\\n\")\nfor i in list(metadata.keys()):\n print(\" - sample and number of elements in {} :- \".format(i),len(list(metadata[i])))\n print(\"\\t\",list(metadata[i])[0], end = \"\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-02-20T10:58:09.753354Z","iopub.execute_input":"2024-02-20T10:58:09.753738Z","iopub.status.idle":"2024-02-20T10:58:15.812047Z","shell.execute_reply.started":"2024-02-20T10:58:09.753692Z","shell.execute_reply":"2024-02-20T10:58:15.811129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(join(TEST,META),\"r\", encoding = \"ISO-8859-1\") as file:\n metadata_test = json.load(file)\nprint(\"Metadata has {} sections. These section has all the Information regarding Images in dataset like class, id, size etc. \".format(len(list(metadata_test.keys()))))\nprint(\"Let us see al the sections in metadata:- \", [print(\" - \",i) for i in list(metadata_test.keys())])\n\nprint(\"Number of Images in our Training set is:- \", len(metadata_test[\"images\"]))\nprint(\"\\n Let us see how every section of Dataset Looks like:-\\n\")\nfor i in list(metadata_test.keys()):\n print(\" - sample and number of elements in {} :- \".format(i),len(list(metadata_test[i])))\n print(\"\\t\",list(metadata_test[i])[0], end = \"\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-02-20T10:58:15.813647Z","iopub.execute_input":"2024-02-20T10:58:15.813936Z","iopub.status.idle":"2024-02-20T10:58:16.277864Z","shell.execute_reply.started":"2024-02-20T10:58:15.813897Z","shell.execute_reply":"2024-02-20T10:58:16.276870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img = pd.DataFrame(metadata['images'])\ntrain_ann = pd.DataFrame(metadata['annotations'])\ntrain_df = pd.merge(train_ann, train_img, left_on='image_id', right_on='id', how='left').drop('image_id', axis=1).sort_values(by=['category_id'])\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T10:58:17.235808Z","iopub.execute_input":"2024-02-20T10:58:17.236163Z","iopub.status.idle":"2024-02-20T10:58:23.105750Z","shell.execute_reply.started":"2024-02-20T10:58:17.236121Z","shell.execute_reply":"2024-02-20T10:58:23.104847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"im = Image.open(\"../input/herbarium-2020-fgvc7/nybg2020/train/images/156/72/354106.jpg\")\nprint(\"Category Id is 15672 and Image Id is 354106 is shown below\")\nim","metadata":{"execution":{"iopub.status.busy":"2024-02-20T10:58:23.107916Z","iopub.execute_input":"2024-02-20T10:58:23.108272Z","iopub.status.idle":"2024-02-20T10:58:23.318473Z","shell.execute_reply.started":"2024-02-20T10:58:23.108219Z","shell.execute_reply":"2024-02-20T10:58:23.317664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"size_of_img = (40, 40)\nfig=plt.figure(figsize=(80,80))\nfor i in range(60):\n ax=fig.add_subplot(20,20,i+1)\n img = cv2.imread(TRAIN + metadata[\"images\"][i][\"file_name\"])\n img = cv2.resize(img,size_of_img)\n ax.imshow(img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-20T10:58:23.319763Z","iopub.execute_input":"2024-02-20T10:58:23.320051Z","iopub.status.idle":"2024-02-20T10:58:34.484485Z","shell.execute_reply.started":"2024-02-20T10:58:23.320012Z","shell.execute_reply":"2024-02-20T10:58:34.483610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nstart_time = time.time()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T10:58:34.486253Z","iopub.execute_input":"2024-02-20T10:58:34.486714Z","iopub.status.idle":"2024-02-20T10:58:34.491099Z","shell.execute_reply.started":"2024-02-20T10:58:34.486658Z","shell.execute_reply":"2024-02-20T10:58:34.490214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n if filename.endswith('.jpg'):\n break\n print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2024-02-20T10:58:34.494397Z","iopub.execute_input":"2024-02-20T10:58:34.494807Z","iopub.status.idle":"2024-02-20T11:06:08.897014Z","shell.execute_reply.started":"2024-02-20T10:58:34.494755Z","shell.execute_reply":"2024-02-20T11:06:08.896134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/herbarium-2020-fgvc7/sample_submission.csv')\ndisplay(sample_sub)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:08.899350Z","iopub.execute_input":"2024-02-20T11:06:08.899681Z","iopub.status.idle":"2024-02-20T11:06:08.960639Z","shell.execute_reply.started":"2024-02-20T11:06:08.899629Z","shell.execute_reply":"2024-02-20T11:06:08.959707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json, codecs\nwith codecs.open(\"../input/herbarium-2020-fgvc7/nybg2020/train/metadata.json\", 'r',\n encoding='utf-8', errors='ignore') as f:\n train_meta = json.load(f)\n \nwith codecs.open(\"../input/herbarium-2020-fgvc7/nybg2020/test/metadata.json\", 'r',\n encoding='utf-8', errors='ignore') as f:\n test_meta = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:08.961840Z","iopub.execute_input":"2024-02-20T11:06:08.962130Z","iopub.status.idle":"2024-02-20T11:06:13.414530Z","shell.execute_reply.started":"2024-02-20T11:06:08.962090Z","shell.execute_reply":"2024-02-20T11:06:13.413694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_meta.keys())","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:13.415928Z","iopub.execute_input":"2024-02-20T11:06:13.416372Z","iopub.status.idle":"2024-02-20T11:06:13.422669Z","shell.execute_reply.started":"2024-02-20T11:06:13.416274Z","shell.execute_reply":"2024-02-20T11:06:13.421700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.DataFrame(train_meta['annotations'])\ndisplay(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:13.423977Z","iopub.execute_input":"2024-02-20T11:06:13.424297Z","iopub.status.idle":"2024-02-20T11:06:15.915651Z","shell.execute_reply.started":"2024-02-20T11:06:13.424244Z","shell.execute_reply":"2024-02-20T11:06:15.914679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cat = pd.DataFrame(train_meta['categories'])\ntrain_cat.columns = ['family', 'genus', 'category_id', 'category_name']\ndisplay(train_cat)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:15.917005Z","iopub.execute_input":"2024-02-20T11:06:15.917432Z","iopub.status.idle":"2024-02-20T11:06:15.986221Z","shell.execute_reply.started":"2024-02-20T11:06:15.917371Z","shell.execute_reply":"2024-02-20T11:06:15.985294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_img = pd.DataFrame(train_meta['images'])\ntrain_img.columns = ['file_name', 'height', 'image_id', 'license', 'width']\ndisplay(train_img)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:15.987426Z","iopub.execute_input":"2024-02-20T11:06:15.987711Z","iopub.status.idle":"2024-02-20T11:06:18.735039Z","shell.execute_reply.started":"2024-02-20T11:06:15.987671Z","shell.execute_reply":"2024-02-20T11:06:18.734146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_reg = pd.DataFrame(train_meta['regions'])\ntrain_reg.columns = ['region_id', 'region_name']\ndisplay(train_reg)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:18.736234Z","iopub.execute_input":"2024-02-20T11:06:18.736558Z","iopub.status.idle":"2024-02-20T11:06:18.746208Z","shell.execute_reply.started":"2024-02-20T11:06:18.736515Z","shell.execute_reply":"2024-02-20T11:06:18.745401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.merge(train_cat, on='category_id', how='outer')\ntrain_df = train_df.merge(train_img, on='image_id', how='outer')\ntrain_df = train_df.merge(train_reg, on='region_id', how='outer')","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:18.747425Z","iopub.execute_input":"2024-02-20T11:06:18.747705Z","iopub.status.idle":"2024-02-20T11:06:20.426642Z","shell.execute_reply.started":"2024-02-20T11:06:18.747667Z","shell.execute_reply":"2024-02-20T11:06:20.425757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.info())\ndisplay(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:20.427967Z","iopub.execute_input":"2024-02-20T11:06:20.428314Z","iopub.status.idle":"2024-02-20T11:06:21.162398Z","shell.execute_reply.started":"2024-02-20T11:06:20.428253Z","shell.execute_reply":"2024-02-20T11:06:21.161521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"na = train_df.file_name.isna()\nkeep = [x for x in range(train_df.shape[0]) if not na[x]]\ntrain_df = train_df.iloc[keep]","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:21.163971Z","iopub.execute_input":"2024-02-20T11:06:21.164316Z","iopub.status.idle":"2024-02-20T11:06:54.539219Z","shell.execute_reply.started":"2024-02-20T11:06:21.164252Z","shell.execute_reply":"2024-02-20T11:06:54.538433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes = ['int32', 'int32', 'int32', 'int32', 'object', 'object', 'object', 'object', 'int32', 'int32', 'int32', 'object']\nfor n, col in enumerate(train_df.columns):\n train_df[col] = train_df[col].astype(dtypes[n])\nprint(train_df.info())\ndisplay(train_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:54.540641Z","iopub.execute_input":"2024-02-20T11:06:54.541058Z","iopub.status.idle":"2024-02-20T11:06:55.286356Z","shell.execute_reply.started":"2024-02-20T11:06:54.541001Z","shell.execute_reply":"2024-02-20T11:06:55.285599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.DataFrame(test_meta['images'])\ntest_df.columns = ['file_name', 'height', 'image_id', 'license', 'width']\nprint(test_df.info())\ndisplay(test_df)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:55.287479Z","iopub.execute_input":"2024-02-20T11:06:55.287812Z","iopub.status.idle":"2024-02-20T11:06:55.680596Z","shell.execute_reply.started":"2024-02-20T11:06:55.287757Z","shell.execute_reply":"2024-02-20T11:06:55.679820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_csv('full_train_data.csv', index=False)\ntest_df.to_csv('full_test_data.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:06:55.681840Z","iopub.execute_input":"2024-02-20T11:06:55.682133Z","iopub.status.idle":"2024-02-20T11:07:14.500013Z","shell.execute_reply.started":"2024-02-20T11:06:55.682095Z","shell.execute_reply":"2024-02-20T11:07:14.499110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Total Unique Values for each columns:\")\nprint(\"{0:10s} \\t {1:10d}\".format('train_df', len(train_df)))\nfor col in train_df.columns:\n print(\"{0:10s} \\t {1:10d}\".format(col, len(train_df[col].unique())))","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:07:14.501371Z","iopub.execute_input":"2024-02-20T11:07:14.501693Z","iopub.status.idle":"2024-02-20T11:07:15.378597Z","shell.execute_reply.started":"2024-02-20T11:07:14.501650Z","shell.execute_reply":"2024-02-20T11:07:15.377790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"family = train_df[['family', 'genus', 'category_name']].groupby(['family', 'genus']).count()\ndisplay(family.describe())","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:07:15.379798Z","iopub.execute_input":"2024-02-20T11:07:15.380146Z","iopub.status.idle":"2024-02-20T11:07:15.687593Z","shell.execute_reply.started":"2024-02-20T11:07:15.380093Z","shell.execute_reply":"2024-02-20T11:07:15.686740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Dense, Dropout, Conv2D, MaxPool2D, Flatten, BatchNormalization, Input, concatenate\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.utils import plot_model\nfrom sklearn.model_selection import train_test_split as tts\n\nin_out_size = (120*120) + 3 #We will resize the image to 120*120 and we have 3 outputs\ndef xavier(shape, dtype=None):\n return np.random.rand(*shape)*np.sqrt(1/in_out_size)\n\ndef fg_model(shape, lr=0.001):\n '''Family-Genus model receives an image and outputs two integers indicating both the family and genus index.'''\n i = Input(shape)\n \n x = Conv2D(3, (3, 3), activation='relu', padding='same', kernel_initializer=xavier)(i)\n x = Conv2D(3, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(x)\n x = MaxPool2D(pool_size=(3, 3), strides=(3,3))(x)\n x = BatchNormalization()(x)\n x = Dropout(0.5)(x)\n x = Conv2D(16, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(x)\n #x = Conv2D(16, (5, 5), activation='relu', padding='same', kernel_initializer=xavier)(x)\n x = MaxPool2D(pool_size=(5, 5), strides=(5,5))(x)\n x = BatchNormalization()(x)\n x = Dropout(0.5)(x)\n x = Flatten()(x)\n \n o1 = Dense(310, activation='softmax', name='family', kernel_initializer=xavier)(x)\n \n o2 = concatenate([o1, x])\n o2 = Dense(3678, activation='softmax', name='genus', kernel_initializer=xavier)(o2)\n \n o3 = concatenate([o1, o2, x])\n o3 = Dense(32094, activation='softmax', name='category_id', kernel_initializer=xavier)(o3)\n \n x = Model(inputs=i, outputs=[o1, o2, o3])\n \n opt = Adam(lr=lr, amsgrad=True)\n x.compile(optimizer=opt, loss=['sparse_categorical_crossentropy', \n 'sparse_categorical_crossentropy', \n 'sparse_categorical_crossentropy'],\n metrics=['accuracy'])\n return x\n\nmodel = fg_model((120, 120, 3))\nmodel.summary()\nplot_model(model, to_file='full_model_plot.png', show_shapes=True, show_layer_names=True)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:07:15.688754Z","iopub.execute_input":"2024-02-20T11:07:15.689095Z","iopub.status.idle":"2024-02-20T11:07:31.266774Z","shell.execute_reply.started":"2024-02-20T11:07:15.689046Z","shell.execute_reply":"2024-02-20T11:07:31.265112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\ntrain_datagen = ImageDataGenerator(featurewise_center=False,\n featurewise_std_normalization=False,\n rotation_range=180,\n width_shift_range=0.1,\n height_shift_range=0.1,\n zoom_range=0.2)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:07:31.268712Z","iopub.execute_input":"2024-02-20T11:07:31.269123Z","iopub.status.idle":"2024-02-20T11:07:31.279174Z","shell.execute_reply.started":"2024-02-20T11:07:31.269069Z","shell.execute_reply":"2024-02-20T11:07:31.278178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = train_df[['file_name', 'family', 'genus', 'category_id']]\nfam = m.family.unique().tolist()\nm.family = m.family.map(lambda x: fam.index(x))\ngen = m.genus.unique().tolist()\nm.genus = m.genus.map(lambda x: gen.index(x))\ndisplay(m)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:07:31.280805Z","iopub.execute_input":"2024-02-20T11:07:31.281154Z","iopub.status.idle":"2024-02-20T11:08:03.199688Z","shell.execute_reply.started":"2024-02-20T11:07:31.281110Z","shell.execute_reply":"2024-02-20T11:08:03.198460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, verif = tts(m, test_size=0.2, shuffle=True, random_state=17)\ntrain = train[:49000]\nverif = verif[:1000]\nshape = (120, 120, 3)\nepochs = 2\nbatch_size = 32\n\nmodel = fg_model(shape, 0.007)\n\n#Disable the last two output layers for training the Family\nfor layers in model.layers:\n if layers.name == 'genus' or layers.name=='category_id':\n layers.trainable = False\n\n#Train Family for 2 epochs\nmodel.fit_generator(train_datagen.flow_from_dataframe(dataframe=train,\n directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n x_col=\"file_name\",\n y_col=[\"family\", \"genus\", \"category_id\"],\n target_size=(120, 120),\n batch_size=batch_size,\n class_mode='multi_output'),\n validation_data=train_datagen.flow_from_dataframe(\n dataframe=verif,\n directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n x_col=\"file_name\",\n y_col=[\"family\", \"genus\", \"category_id\"],\n target_size=(120, 120),\n batch_size=batch_size,\n class_mode='multi_output'),\n epochs=epochs,\n steps_per_epoch=len(train)//batch_size,\n validation_steps=len(verif)//batch_size,\n verbose=1,\n workers=8,\n use_multiprocessing=False)\n\n#Reshuffle the inputs\ntrain, verif = tts(m, test_size=0.2, shuffle=True, random_state=17)\ntrain = train[:40000]\nverif = verif[:10000]\n\n#Make the Genus layer Trainable\nfor layers in model.layers:\n if layers.name == 'genus':\n layers.trainable = True\n \n#Train Family and Genus for 2 epochs\nmodel.fit_generator(train_datagen.flow_from_dataframe(dataframe=train,\n directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n x_col=\"file_name\",\n y_col=[\"family\", \"genus\", \"category_id\"],\n target_size=(120, 120),\n batch_size=batch_size,\n class_mode='multi_output'),\n validation_data=train_datagen.flow_from_dataframe(\n dataframe=verif,\n directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n x_col=\"file_name\",\n y_col=[\"family\", \"genus\", \"category_id\"],\n target_size=(120, 120),\n batch_size=batch_size,\n class_mode='multi_output'),\n epochs=epochs,\n steps_per_epoch=len(train)//batch_size,\n validation_steps=len(verif)//batch_size,\n verbose=1,\n workers=8,\n use_multiprocessing=False)\n\n#Reshuffle the inputs\ntrain, verif = tts(m, test_size=0.2, shuffle=True, random_state=17)\ntrain = train[:40000]\nverif = verif[:10000]\n\n#Make the category_id layer Trainable\nfor layers in model.layers:\n if layers.name == 'category_id':\n layers.trainable = True\n \n#Train them all for 2 epochs\nmodel.fit_generator(train_datagen.flow_from_dataframe(dataframe=train,\n directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n x_col=\"file_name\",\n y_col=[\"family\", \"genus\", \"category_id\"],\n target_size=(120, 120),\n batch_size=batch_size,\n class_mode='multi_output'),\n validation_data=train_datagen.flow_from_dataframe(\n dataframe=verif,\n directory='../input/herbarium-2020-fgvc7/nybg2020/train/',\n x_col=\"file_name\",\n y_col=[\"family\", \"genus\", \"category_id\"],\n target_size=(120, 120),\n batch_size=batch_size,\n class_mode='multi_output'),\n epochs=epochs,\n steps_per_epoch=len(train)//batch_size,\n validation_steps=len(verif)//batch_size,\n verbose=1,\n workers=8,\n use_multiprocessing=False)","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:08:03.203130Z","iopub.execute_input":"2024-02-20T11:08:03.203469Z","iopub.status.idle":"2024-02-20T11:46:55.220816Z","shell.execute_reply.started":"2024-02-20T11:08:03.203417Z","shell.execute_reply":"2024-02-20T11:46:55.219802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('/kaggle/working/fg_model.h5')","metadata":{"execution":{"iopub.status.busy":"2024-02-20T11:46:55.222843Z","iopub.execute_input":"2024-02-20T11:46:55.223270Z","iopub.status.idle":"2024-02-20T11:47:00.384057Z","shell.execute_reply.started":"2024-02-20T11:46:55.223205Z","shell.execute_reply":"2024-02-20T11:47:00.383177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}