{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport os\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers,models,Input,Model\nfrom tensorflow.keras.layers import Dense, Flatten,GlobalAveragePooling2D,Dropout\nfrom tensorflow.keras.layers.experimental import preprocessing\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn import svm\nfrom sklearn.preprocessing import normalize\n\nfrom kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Parameters\nDIR_INPUT = '/kaggle/input/cassava-leaf-disease-classification/'\nDIR_TRAIN = '/kaggle/input/cassava-leaf-disease-classification/train_images/'\nDIR_TEST = '/kaggle/input/cassava-leaf-disease-classification/test_images/'\nDIR_TRAIN_RECORD = '/kaggle/input/cassava-leaf-disease-classification/train_tfrecords/'\nDIR_TEST_RECORD = '/kaggle/input/cassava-leaf-disease-classification/test_tfrecords/'\nDIR_JSON = DIR_INPUT+'/label_num_to_disease_map.json'\nSAVED_WEIGHTS='../input/saved-output-weights/inception_3.h5'\n#GCS_PATH = KaggleDatasets().get_gcs_path()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\n# N_FOLDS = 1\nBATCH_SIZE_GEN = 32\nBATCH_SIZE_FIT = 32\nBATCH_SIZE_TEST = 64\n\n# IMG_SIZE = (299,299,3) # Inception size\nIMG_SIZE = (300,300,3)\nEPOCH = 25","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocess\nPrint Image to see how they look,\nConvert all to single size using padding\nNormalize color intensities\nTest label data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train = pd.read_csv(DIR_INPUT+'/train.csv')\ntrain = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/train.csv')\ntrain['label']=train['label'].astype(str)\n\n\ntest =[]\nfor dirname, _, filenames in os.walk(DIR_TEST):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        test.append([filename,-1])\ntest = pd.DataFrame(test,columns = ['image_id','label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"disp=train.groupby(by='label').agg('count')\ndisp['image_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_resampled = train\ndf_resampled = train_test_split(train_size=0.2,stratify=df_resampled['label'])\ndf_resampled = df_resampled.sample(frac=1).reset_index(drop=True)\n# df_resampled","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploratory Data Analysis\ncount images per class in labels\nrandomly print few per class\nLook at few images of each class"},{"metadata":{"trusted":true},"cell_type":"code","source":"# for l in ['0','1','2','3','4']:\n#     plt.figure(figsize=(20,60))\n#     for i in range(30):\n#         sample = train[train['label']==l]\n#         print_index = int(np.random.randint(len(sample)))\n#         plt.subplot(10,3,i+1)\n#         image = plt.imread(DIR_TRAIN+str(sample.iloc[print_index,0]))\n#         plt.xlabel(print_index)\n#         plt.ylabel(sample.iloc[print_index,1])\n#         plt.imshow(image)\n#     plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Setup\nWhile any pretrained nets can be used, let's try to train our own simple CNN first. CNN - 32Kernel, 55 + maxpool 22 X 2 times 64Kernel, 33 + maxpool 22 X 2 times 128Kernel, 33 + maxpool 22 X 2 times - relu Dense network X 2 times - 5 classes - softmax activation ADAM, Loss: categorical loss, Epoch - 10, Acuracy, what about linearity? regularization, RESNET, batch norm, random init etc? Inception, transformer?\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(\n        validation_split=0.1)\n\ntrain_generator = train_datagen.flow_from_dataframe(\n        df_resampled,DIR_TRAIN,x_col='image_id',y_col='label',\n#         target_size=(299,299),interpolation='bilinear',\n        batch_size=BATCH_SIZE_GEN,\n        subset ='training',    \n        class_mode='sparse', shuffle= False)#,seed=SEED)\n\nval_generator = train_datagen.flow_from_dataframe(\n        df_resampled,DIR_TRAIN,x_col='image_id',y_col='label',\n#         target_size=(299,299),interpolation='bilinear',\n        batch_size=BATCH_SIZE_GEN,shuffle=False,\n        subset ='validation',\n        class_mode='sparse')#, shuffle= True,seed=SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Pre Processing using Tensors\n\ndata_augmentation = tf.keras.Sequential(\n    [\n        preprocessing.Resizing(IMG_SIZE[0],IMG_SIZE[1],interpolation='bicubic'),\n        preprocessing.RandomFlip(\"horizontal_and_vertical\"),\n        \n# # #         preprocessing.RandomRotation(0.5,fill_mode='constant'),\n        preprocessing.RandomZoom((-0.2,0)),\n#         preprocessing.Rescaling(1.0 / 255) # Rescaling not required. EFfnet has it inbuilt.\n    ]\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model EfficientNet"},{"metadata":{"trusted":true},"cell_type":"code","source":"# base_model = tf.keras.applications.InceptionV3(include_top=False, weights='../input/inception-v3-weights/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5', input_shape=IMG_SIZE)\nbase_model = tf.keras.applications.EfficientNetB3(include_top=False, weights='../input/keras-efficientnetb3-no-top-weights/efficientnetb3_notop.h5',input_shape=IMG_SIZE,drop_connect_rate=0.4)\nbase_model.trainable = True\ninputs = Input(shape=IMG_SIZE)\nx = data_augmentation(inputs)\nx = base_model(x)\nx = GlobalAveragePooling2D()(x)\nx = Flatten()(x)\nx = Dropout(0.5)(x)\n# x = Dense(1024, activation=\"relu\", dtype='float64')(x)\n# x = Dropout(0.3)(x)\n# x = Dense(1024, activation=\"relu\", dtype='float64')(x)\n# x = Dropout(0.3)(x)\n# x = Dense(256, activation=\"relu\", dtype='float64')(x)\n# x = Dropout(0.2)(x)\noutputs = Dense(5, activation=\"softmax\", dtype='float64')(x)\nmodel = Model(inputs=inputs, outputs=outputs)\nopt = tf.keras.optimizers.SGD(0.005)#clipvalue=10.0)\n\nmodel.compile(optimizer =opt,loss='sparse_categorical_crossentropy',metrics=['accuracy'])\n# Train model normally\nmodel.summary()\n# base_model.summary()\n# model.features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau, EarlyStopping,LearningRateScheduler\n\ndef schedule (epoch,lr):\n    max_lr = 0.005\n    min_lr = 0.0005\n    max_epoch = EPOCH\n    if epoch<=max_epoch/2:\n        lr = (epoch)*(max_lr-min_lr)/((max_epoch)/2-1)+min_lr\n\n    else:\n        lr = (epoch-max_epoch/2)*(min_lr-max_lr)/(max_epoch/2) + max_lr\n    return lr\n\ncallbacks_list_default = [\n#         # reduce learning rate by a factor of 5 (i.e. lr/=5.0) \n#         # if val_loss does not reduce for 3 epochs\n#         ReduceLROnPlateau(\n#             monitor='val_loss',\n#             factor=0.5,\n#             patience=5,\n#             verbose=1\n#         ),\n        # Stop training if val_acc does not improve for\n        # 5 or more epochs\n        EarlyStopping(\n            monitor='val_accuracy',\n            patience=7,\n            verbose=1        \n        ),\n    tf.keras.callbacks.LearningRateScheduler(schedule, verbose=1)\n    ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n#         train_dataset,\n        train_generator,\n        batch_size=BATCH_SIZE_FIT,\n#         steps_per_epoch=int(len(train)/128),\n        epochs=EPOCH,\n        validation_data=val_generator,\n    callbacks =callbacks_list_default\n)\n\nmodel.save('./Efficientnet.h5')\nmodel.save_weights('./Efficientnet_weights.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# EValuate feature prediction \ny_pred = np.argmax(model.predict(train_generator,verbose=1),axis=1)\nacc = np.sum(y_pred == train_generator.classes)/len(y_pred)\nprint(\"Debug Train - Test acccuracy is :\",acc)\nprint(\"Number of test samples :\",len(y_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model_EfficientNet"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setup SVM model\nJUST = './Efficientnet.h5'\n\nbase_model_svm = models.load_model(JUST)\nlayer = base_model_svm.layers[3].name\noutputs = base_model_svm.get_layer(layer).output\nmodel_svm = Model(inputs = base_model_svm.input,outputs=outputs)\nmodel_svm.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train Subset Fiting\n\nfeatures = base_model_svm.predict(train_generator,verbose=1)\nfeatures_norm = normalize(features, norm='l2', axis=0, copy=True, return_norm=False)\n\nprint(\"Extracted Features\")\nsvm_model = svm.LinearSVC(C=1, verbose=1, max_iter=100000, loss='squared_hinge', penalty='l2', dual=False )\n# svm_model = svm.SVC(kernel='rbf',probability=True,C=0.9,verbose=1)\nsvm_model.fit(features_norm,train_generator.classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train Subset Performance\nprint(\"Fitted Features. Predicting Training subset\")\npreds = svm_model.predict(features_norm)\nprint(sum(preds==train_generator.classes)/len(preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Val Subset Performance\nfeatures_val = base_model_svm.predict(val_generator,verbose=1)\nfeatures_val_norm = normalize(features_val, norm='l2', axis=0, copy=True, return_norm=False)\n\nprint(\"Fitted Features. Predicting validation\")\npreds = svm_model.predict(features_val_norm)\nprint(sum(preds==val_generator.classes)/len(preds))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# history.history['val_accuracy']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Loss, Accuracy VS EPOCHs/steps\n\n# plt.figure()\n# plt.plot(history.history['loss'],'bo-')\n# plt.plot(history.history['val_loss'],'ro-')\n# plt.legend(['loss','val_loss'])\n# plt.figure()\n# plt.plot(history.history['accuracy'],'bo-')\n# plt.plot(history.history['val_accuracy'],'ro-')\n# plt.legend(['accuracy','val_accuracy'])\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PREDICTION"},{"metadata":{"trusted":true},"cell_type":"code","source":"# JUST = '../input/march20-1/Efficientnet.h5'\n# model = models.load_model(JUST)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def do_error(n = 25):\n#     plt.figure(figsize=(20,20))\n#     # Show N samples of miss classifications\n#     train_image_id = []\n#     print(\"Do Missclassfication Analysis\")\n#     DIR = DIR_TRAIN\n            \n#     train_perf = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/train.csv')\n#     train_perf['label']=train_perf['label'].astype(str)\n#     train_perf = train_perf[0:500]\n#     train_perf_datagen = ImageDataGenerator()\n#     train_perf_generator = train_perf_datagen.flow_from_dataframe(\n#             train_perf,DIR,x_col='image_id',y_col='label',\n#             target_size=(300,300),interpolation='bilinear',\n#             batch_size=BATCH_SIZE_TEST,class_mode='sparse', shuffle=False\n#             )\n    \n#     y_pred = np.argmax(model.predict(train_perf_generator,verbose=1),axis=1)\n#     y_pred = y_pred.astype(str)\n    \n#     error_index = np.where(y_pred != list(train_perf['label']))[0]\n    \n\n#     for i in range(n):\n#         ind = error_index[i]\n#         plt.subplot((int(n/5)+1),5,i+1)\n#         plt.tick_params( which='both', bottom=False, left=False,top=False,labelbottom=False, labelleft=False) \n#         # Get the class names from labels\n#         img = plt.imread(DIR+'/'+train_perf.loc[ind,'image_id'])\n#         plt.xlabel('Pred: '+y_pred[ind])\n#         plt.title(train_perf.loc[ind,'label'])\n#         plt.imshow(img)\n#         plt.tight_layout()\n\n# do_error()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Train performance\n# train_image_id = []\n# print(\"Debug On. Running on test on train dataset\")\n# DIR = DIR_TRAIN\n        \n# train_perf = pd.read_csv('/kaggle/input/cassava-leaf-disease-classification/train.csv')\n# train_perf['label']=train_perf['label'].astype(str)\n# train_perf_datagen = ImageDataGenerator()\n# train_perf_generator = train_perf_datagen.flow_from_dataframe(\n#         train_perf,DIR,x_col='image_id',y_col='label',\n#         target_size=(300,300),interpolation='bilinear',\n#         batch_size=BATCH_SIZE_TEST,class_mode='sparse', shuffle=False\n#         )\n\n# y_pred = np.argmax(model.predict(train_perf_generator,verbose=1),axis=1)\n# y_pred = y_pred.astype(str)\n# acc = np.sum(y_pred == list(train_perf['label']))/len(y_pred)\n# print(\"Debug Train - Test acccuracy is :\",acc)\n# print(\"Number of test samples :\",len(y_pred))\n\n# submit = pd.DataFrame({ 'label': y_pred.astype(str)})\n# print(submit.groupby('label').size())\n# print(submit)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print('Confusion Matrix of validation data')\n# print(confusion_matrix(train_perf['label'], y_pred.astype(str)))\n# print('Classification Report')\n# print(classification_report(train_perf['label'], y_pred.astype(str), target_names=['0','1','2','3','4']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.evaluate(train_perf_generator,use_multiprocessing=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test performance \ntest_image_id = []\nDIR = DIR_TEST\nfor dirname, _, filenames in os.walk(DIR):\n    for filename in filenames:\n        test_image_id.append(filename)\nprint('Number of Test files is: ',len(test_image_id))\nlabel = np.zeros(len(test_image_id))\ntest = pd.DataFrame({'image_id':test_image_id,'label':label})\ntest['label']=test['label'].astype(str)\n\ntest_datagen = ImageDataGenerator()\ntest_generator = test_datagen.flow_from_dataframe(\n        test,DIR,x_col='image_id',\n        target_size=(300,300),interpolation='bicubic',\n        batch_size=BATCH_SIZE_TEST,class_mode=None,shuffle=False\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_test = base_model_svm.predict(test_generator)\nfeatures_test_norm = normalize(features_test, norm='l2', axis=0, copy=True, return_norm=False)\n\ny_pred = svm_model.predict(features_test_norm)\ny_pred = y_pred.astype(str)\nprint(\"Number of test samples :\",len(y_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SUBMISSION"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred=y_pred.astype(int)\n\nsubmit = pd.DataFrame({ 'image_id': test['image_id'], 'label': y_pred})\nsubmit.to_csv(\"./submission.csv\", index=False)\n\nreader =pd.read_csv('./submission.csv')\nreader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n# Error Analysis\n"},{"metadata":{},"cell_type":"markdown","source":"Version 41: \n    * EffNetB3 Model 0.4 Drop, 1024 + 1024 (0.3 each) + 256 (0.2 dropout) 3 FC added for classification.\n    * SGD 0.01 - 0.0005 Inv V LR scheduler. Added early stopping. \n    * Over sampling removed\n    * Epoch 20\n    * Accuracy ?\n    \nVersion 40: \n    * Base T is False. EffNetB3 Model 0.4 Drop, 1024 + 1024 (0.3 each) + 256 (0.2 dropout) 3 FC added for classification.\n    * SGD 0.01 - 0.0005 Inv V LR scheduler. Added early stopping. \n    * Over sampling removed\n    * Epoch 20\n    * Accuracy 72%\n    \nVersion 39: \n    * EffNetB3 Model 0.3 Drop, 1024 + 1024 (0.3 each) 2 FC added.\n    * SGD 0.005 - 0.0005 Inv V LR scheduler. Added early stopping and reduceLr on Plateau\n    * Over sampling removed\n    * Epoch 35\n    * Accuracy 82%. (In train set, Training saturates at 94, validation at 83)\n\nVersion 39: \n    * EffNetB3 Model 0.3 Drop, 1024 + 1024 (0.3 each) 2 FC added.\n    * SGD 0.005 - 0.0005 Inv V LR scheduler. Added early stopping and reduceLr on Plateau\n    * Over sampling removed\n    * Epoch 35\n    * Accuracy 82%\n\n\nVersion 29: \n    * EffNetB3 Model 0.2 Drop, 1024 + 1024 2 FC added.\n    * SGD 0.005, momentum 0. Added early stoppingand reduceLronPlateau\n    * Over sampling removed\n    * Epoch 80\n    * Accuracy \n\nVersion 28: \n    * EffNetB3 Model 0.2 Drop, 1024 + 1024 2 FC added.\n    * SGD 0.0008, momentum 0. Added early stoppingand reduceLronPlateau\n    * Over sampling removed\n    * Epoch 80\n    * Accuracy 86 Train,82 Val\n    \nVersion 27: \n    * EffNetB3 Model 0.2 Drop, 1024 + 1024 2 FC added.\n    * SGD 0.0003, momentum 0. Added early stoppingand reduceLronPlateau\n    * Over sampling removed\n    * Epoch 40\n    * Accuracy 71 on train data and validation data. Waiting for the hidden test set.\n    \nVersion 23: \n    * EffNetB3 Model 0.2 Drop, 1024 + 1024 2 FC added.\n    * SGD 0.0001, momentum 0  (Removed Loss weights.Keeping it to only drop and SGD changes)\n    * Over sampling removed\n    * Epoch 10\n    * Accuracy 0.63 - May be LR too low. accuracy is stagnant for 20 epochs. \n    \nVersion 21: \n    * EffNetB3 Model 0.2 Drop, 1024 + 1024 2 FC added.\n    * SGD 0.1, momentum 0.3  (Removed Loss weights.Keeping it to only drop and SGD changes)\n    * Over sampling removed\n    * Epoch 10\n    * Accuracy 0.14\n\nVersion 20: \n    * EffNetB3 Model 0.1 Drop, 256 + 256 2 FC added.\n    * SGD 0.1, momentum 0.3  (Removed Loss weights.Keeping it to only drop and SGD changes)\n    * Over sampling removed\n    * Epoch 10\n    * Accuracy 0.14\n    \nVersion 19: \n    * EffNetB3 Model 0.1 Drop, 256 + 256 2 FC added.\n    * SGD 0.1, momentum 0.3  Loss weights = 1,1,1,0.5,1\n    * Over sampling removed\n    * Epoch 10\n    * Accuracy 0.1 on validation not test\n\nVersion 18: \n    * EffNetB3 Model 256 + 256 2 FC added\n    * Over sampling removed\n    * Epoch 10\n    * Accuracy 0.14\n\nVersion 17: \n    * Inception Model 256 + 256 2 FC added\n    * Over sampling removed\n    * Epoch 10\n    * Acc 0.1\n    \nVersion 16: \n    * Inception Model 256 + 32 2 FC added\n    * Over sampling ON\n    * Epoch 2\n    * Accuracy 0.1 on full test, 0.6 on train."},{"metadata":{},"cell_type":"markdown","source":"**Error Log**\nModel Architecture 1:\n1. No dropout, NO class wieghts, 20 Epochs : 0.6 Accuracy\n2. Dropout + Class wieghts, 20 epochs      : 0.1 accuracy\n3. Dropout + No class weights, 20 epochs   : \n\nModel Architecture 2:"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}