{"cells":[{"metadata":{},"cell_type":"markdown","source":" \n

Efficient NEt B5 Kfold over Optuna

\n "},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --quiet ../input/kerasapplications\n!pip install --quiet ../input/efficientnet-source-code","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import libraries\nimport numpy as np \nimport pandas as pd \nfrom os.path import join\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\nimport os\n\nimport IPython.display as display\n\nfrom PIL import Image\nimport warnings\n#warnings.filterwarnings(\"ignore\")\n\nimport random as rd\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import Input,optimizers\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout,BatchNormalization, Activation, Input, GlobalAveragePooling2D, concatenate,Lambda\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping,TensorBoard\nfrom tensorflow.keras.applications import EfficientNetB0,EfficientNetB4,EfficientNetB3, Xception\nimport efficientnet.tfkeras as efn\n\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import confusion_matrix,roc_curve,multilabel_confusion_matrix\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import accuracy_score\nfrom tqdm import tqdm\nimport itertools\n\nimport optuna\n\ncolors = plt.rcParams['axes.prop_cycle'].by_key()['color']\ndef plot_confusion_matrix(y_true, y_pred, class_names,title=\"Confusion matrix\",normalize=False,onehot = False):\n \"\"\"\n Returns a matplotlib figure containing the plotted confusion matrix.\n Args:\n cm (array, shape = [n, n]): a confusion matrix of integer classes\n class_names (array, shape = [n]): String names of the integer classes\n \"\"\"\n if onehot :\n cm = confusion_matrix([y_i.argmax() for y_i in y_true], [y_ip.argmax() for y_ip in y_pred])\n else:\n cm = confusion_matrix(y_true, y_pred)\n figure = plt.figure(figsize=(8, 8))\n plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n plt.title(title)\n plt.colorbar()\n tick_marks = np.arange(len(class_names))\n plt.xticks(tick_marks, class_names, rotation=45)\n plt.yticks(tick_marks, class_names)\n\n # Normalize the confusion matrix.\n cm = np.around(cm.astype('float') / cm.sum(axis=1)[:, np.newaxis], decimals=2) if normalize else cm\n\n # Use white text if squares are dark; otherwise black.\n threshold = cm.max() / 2.\n for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n color = \"white\" if cm[i, j] > threshold else \"black\"\n plt.text(j, i, cm[i, j], horizontalalignment=\"center\", color=color)\n\n plt.tight_layout()\n plt.ylabel('True label')\n plt.xlabel('Predicted label')\n plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SampleSubmission = \"../input/cassava-leaf-disease-classification/sample_submission.csv\"\nlabels_desease = \"../input/cassava-leaf-disease-classification/label_num_to_disease_map.json\"\ntraintablepath = \"../input/cassava-leaf-disease-classification/train.csv\"\n\ntrain_tfr_path = \"../input/cassava-leaf-disease-classification/train_tfrecords\"\ntest_tfr_path = \"../input/cassava-leaf-disease-classification/train_tfrecords\"\n\ntrain_img_path = \"../input/cassava-leaf-disease-classification/train_images\"\ntest_img_path = \"../input/cassava-leaf-disease-classification/test_images\"\n\nmodels_path = \"\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"img_augmentation = tf.keras.Sequential(\n [\n #tf.keras.layers.experimental.preprocessing.RandomCrop(image_size, image_size), #during inference will work like a resize layer\n #tf.keras.layers.experimental.preprocessing.RandomTranslation((-.1,.1), (-.1,-.1)),\n tf.keras.layers.experimental.preprocessing.RandomZoom((-.1,-.1)),\n tf.keras.layers.experimental.preprocessing.RandomFlip(\"horizontal_and_vertical\"),\n tf.keras.layers.experimental.preprocessing.RandomRotation(0.2),\n tf.keras.layers.experimental.preprocessing.RandomContrast(0.1),\n ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageCustomGenerator(tf.keras.utils.Sequence):\n \"\"\"\n df_table, dataframe\n col_name, column name for file images\n col_label, column label\n indexs, index to extract from dataframe\n path_files, folder path where images are stored\n batch_size,\n target_size, output images\n \"\"\"\n def __init__(self, \n df_table,\n col_name,\n col_label,\n path_files,\n batch_size,\n target_size,\n augmented_seq=None,\n indexs=None,\n shuffle = False,\n TEST = False\n ):\n self.df_table = df_table\n self.augmented_seq = augmented_seq\n self.indexs = indexs if not TEST else df_table.index.tolist()\n self.col_name = col_name\n self.col_label = col_label\n self.path_files = path_files\n self.batch_size = batch_size\n self.target_size = target_size\n self.shuffle = shuffle\n self.TEST = TEST\n self.n_class = df_table[col_label].nunique() if not TEST else 1\n self.on_epoch_end()\n self.__info()\n \n def __info(self):\n print(\"Custom Image generator {:4d} images\".format(len(self.indexs)))\n\n def __len__(self):\n #batches per epochs\n return len(self.indexs) // self.batch_size + 1 if len(self.indexs)%self.batch_size != 0 else len(self.indexs) // self.batch_size\n\n def __getitem__(self, index):\n indexs_step = self.index_gen[index * self.batch_size:(index + 1) * self.batch_size]\n batch = [self.indexs[k] for k in indexs_step]\n X, y = self.__get_data(batch)\n return X, y\n\n def on_epoch_end(self):\n self.index_gen = np.arange(len(self.indexs))\n if self.shuffle == True and self.TEST == False:\n np.random.shuffle(self.index_gen)\n \n def __get_data(self, batch):\n X = []\n if not self.TEST:\n y = self.df_table.iloc[batch,:][self.col_label].to_numpy().astype(int)\n file_list = self.df_table.iloc[batch,:][self.col_name]\n file_list = file_list.map(lambda x: os.path.join(self.path_files,x))\n\n for name in file_list:\n img_ = tf.image.resize(np.array(np.array(Image.open(name))), self.target_size, method = \"bilinear\")\n X.append(img_)\n return tf.stack(X), tf.stack(y)\n \n else:#################### TEST BEHAVIOR ################\n file_list = self.df_table.iloc[batch,:][self.col_name]\n file_list = file_list.map(lambda x: os.path.join(self.path_files,x))\n for name in file_list:\n img_ = tf.image.resize(np.array(np.array(Image.open(name))),[600,800], method = \"bilinear\")\n img_ = tf.image.random_crop(img_, [*self.target_size,3])\n X.append(img_)\n return self.augmented_seq(tf.stack(X))/255, tf.zeros([len(batch)], tf.float32)\n \n ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EfficientNet Model B5"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(model_id,CLASSES=5,IMAGE_SIZE=(450,450)):\n if model_id==0:\n pretrained_model = tf.keras.applications.Xception(weights = None, input_shape=[*IMAGE_SIZE, 3], include_top=False)\n elif model_id==1:\n pretrained_model = efn.EfficientNetB5(weights = None, include_top = False,input_shape=[*IMAGE_SIZE, 3] )\n elif model_id ==2:\n pretrained_model = efn.EfficientNetB0(weights = None, include_top = False,input_shape=[*IMAGE_SIZE, 3] )\n \n pretrained_model.trainable = True\n model = tf.keras.Sequential([\n pretrained_model,\n GlobalAveragePooling2D(),\n Dropout(0.3),\n Dense(CLASSES, activation='softmax')\n ])\n model.compile(\n optimizer='adam',\n loss = 'categorical_crossentropy',\n metrics=[tf.keras.metrics.CategoricalAccuracy(),\n tf.keras.metrics.PrecisionAtRecall(recall=0.8),\n tf.keras.metrics.AUC(name='auc', multi_label= True),\n tf.keras.metrics.Recall(name='recall')])\n return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating Generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_table = pd.read_csv(traintablepath)\ntrain_table_gen = train_table.copy()\ntrain_table_gen[\"label\"] = train_table_gen[\"label\"].astype(str)\ntrain_datagen = ImageDataGenerator(rescale=1./255,).flow_from_dataframe(\n dataframe = train_table_gen,\n directory = train_img_path,\n x_col=\"image_id\",\n y_col=\"label\",\n target_size=(450, 450),\n batch_size=32,\n shuffle=False,\n class_mode='categorical',\n validate_filenames=False,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_table.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference over the whole dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\npath_model_efnB5 = glob.glob(\"../input/cassava-efficientnetb5-450px/*.hdf5\") \ncorrect = train_table['label']\nresults = []\n\nfor model_k in path_model_efnB5:\n tf.keras.backend.clear_session() \n model = create_model(1)\n model.load_weights(model_k)\n predict_k = model.predict(train_datagen, verbose = 1)\n df = pd.DataFrame(data= predict_k, columns = [\"c0\", \"c1\", \"c2\", \"c3\", \"c4\"])\n results.append(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def total_inference(dfs, w0, w1, w2, w3,w4):\n base= w0 + w1+ w2+ w3+w4\n total = dfs[0]*w0 + dfs[1]*w1 + dfs[2]*w2 + dfs[3]*w3 + dfs[4]*w4\n total /= base\n total = total.to_numpy()\n return np.argmax(total, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_inference = total_inference(results, .2,.2,.2,.2,.2)\naccuracy_score(correct, final_inference)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Optuna Optimization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# You can increase iteration number.\niteration = 4000\n\noptuna.logging.disable_default_handler() # not display log\n#optuna.logging.enable_default_handler() # display log","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndef objective(trial):\n r_min = 0\n r_max = 1\n w0 = trial.suggest_uniform('w0', r_min, r_max)\n w1 = trial.suggest_uniform('w1', r_min, r_max)\n w2 = trial.suggest_uniform('w2', r_min, r_max)\n w3 = trial.suggest_uniform('w3', r_min, r_max)\n w4 = trial.suggest_uniform('w4', r_min, r_max)\n\n pred = total_inference(results, w0, w1, w2, w3,w4)\n score = accuracy_score(correct, pred)\n #print('a:%1.3f,b:%1.3f,c:%1.3f,d:%1.3f,e:%1.3f,score %1.3f' % (a,b,c,d,e,score))\n return score\nSEED=5050\n#study = optuna.create_study(direction='maximize')\nstudy = optuna.create_study(direction='maximize', sampler=optuna.samplers.TPESampler(seed=SEED))\n#study.optimize(objective, n_trials=iteration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study.best_value","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study.best_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([trial.value for trial in study.trials])\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([trial.params['w0'] for trial in study.trials], label='w0')\nplt.plot([trial.params['w1'] for trial in study.trials], label='w1')\nplt.plot([trial.params['w2'] for trial in study.trials], label='w2')\nplt.plot([trial.params['w3'] for trial in study.trials], label='w3')\nplt.plot([trial.params['w4'] for trial in study.trials], label='w4')\nplt.legend()\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from optuna.visualization import plot_optimization_history\nplot_optimization_history(study)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\nimport glob\npath_model_efnB5 = glob.glob(\"../input/cassava-efficientnetb5-450px/*.hdf5\")\n#path_model_xcept = glob.glob(\"../input/cassava-xception-450px/*.hdf5\")\n#path_model_efnB0 = glob.glob(\"../input/cassava-efficientnetb0-450/*.hdf5\")\n# 0.8912931719399916\nweights = {0: 0.0004230151176771213,\n 1: 0.5114438069306837,\n 2: 0.06629172919594499,\n 3: 0.42302383984983993,\n 4: 0.742831400357499}\n\nTTA_n = 4\n\n#table_test_dummy = sampSum.copy()\ntable_test_dummy = pd.DataFrame()\ntable_test_dummy['image_id'] = list(os.listdir(test_img_path))\ntable_test_dummy['label'] = 0\n\nn_splits = 5\n#### Just for testing purpose\ntest_gen = ImageCustomGenerator(\n df_table = table_test_dummy, # sampSum\n col_name = \"image_id\",\n col_label = \"label\",\n augmented_seq = img_augmentation, \n path_files= test_img_path , # test_img_path\n batch_size = 6,\n target_size = (450, 450),\n TEST = True)\n\n\ny_predict = np.zeros((len(table_test_dummy),5)) #np.zeros((len(sampSum),5))\nmodel_name = [ \"Xception\",\"EfficientNet B5\",\"EfficientNet B0\"]\nfor id_model, list_kfold_models in enumerate([path_model_efnB5 ],1):\n y_predict_model = np.zeros((len(table_test_dummy),5))\n print(\"Inference by model - {}\".format(model_name[id_model]))\n for ki, model_kfold_path in enumerate(list_kfold_models):\n print(\"{} - {} Kfold pre trained\".format(model_name[id_model], ki))\n tf.keras.backend.clear_session() \n model = create_model(id_model)\n model.load_weights(model_kfold_path)\n y_predict_tta = np.zeros((len(table_test_dummy),5))\n for t in range(TTA_n):\n print(\"N°{} TTA\".format(t))\n y_predict_tta += model.predict(test_gen, verbose=1)\n \n y_predict_model += y_predict_tta/TTA_n*weights[ki]\n y_predict += y_predict_model\n\n \ny_predict = np.argmax(y_predict, axis = 1)\ntable_test_dummy[\"label\"] = y_predict\ntable_test_dummy.to_csv(\"submission.csv\", index=False)\ntable_test_dummy.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}