{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import usefull library","metadata":{}},{"cell_type":"code","source":"import os\nimport time\nimport shutil\nimport random\nimport cv2\nimport pandas as pd\nimport seaborn as sn\nimport tensorflow as tf\nimport tensorflow_hub as hub\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import classification_report\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.applications.efficientnet import preprocess_input, EfficientNetB5\nfrom tensorflow.keras.models import *\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.utils import *\nfrom tensorflow.keras.callbacks import *\nfrom tensorflow.keras.initializers import *\nfrom kaggle_datasets import KaggleDatasets\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-29T06:00:17.19299Z","iopub.execute_input":"2021-11-29T06:00:17.193727Z","iopub.status.idle":"2021-11-29T06:00:23.606943Z","shell.execute_reply.started":"2021-11-29T06:00:17.19363Z","shell.execute_reply":"2021-11-29T06:00:23.606138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config TPU","metadata":{}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:00:38.941879Z","iopub.execute_input":"2021-11-29T06:00:38.942092Z","iopub.status.idle":"2021-11-29T06:00:44.634527Z","shell.execute_reply.started":"2021-11-29T06:00:38.942063Z","shell.execute_reply":"2021-11-29T06:00:44.633844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set path","metadata":{}},{"cell_type":"code","source":"EPOCHS = 15\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nWIDTH = 480\nHEIGHT = 480\nCHANNELS = 3\nLEARNING_RATE = 0.001\nCLASSES = 6\nSEED = 32\ntop_dropout_rate = 0.2\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:00:44.635374Z","iopub.execute_input":"2021-11-29T06:00:44.636009Z","iopub.status.idle":"2021-11-29T06:00:44.641092Z","shell.execute_reply.started":"2021-11-29T06:00:44.635976Z","shell.execute_reply":"2021-11-29T06:00:44.640295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('fgvc8aug')\nTRAIN_PATH = GCS_DS_PATH + \"/data_full_augmentation_images/data_full_augmentation/images/\"\nprint(GCS_DS_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:00:44.642681Z","iopub.execute_input":"2021-11-29T06:00:44.643162Z","iopub.status.idle":"2021-11-29T06:00:45.030446Z","shell.execute_reply.started":"2021-11-29T06:00:44.643114Z","shell.execute_reply":"2021-11-29T06:00:45.029625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_model = 'EfficientNet_B5_Fullaug.h5'\nhist_path = 'EfficientNet_B5_Fullaug_Hist.log'\ntrain_image = '../input/fgvc8aug/data_full_augmentation_images/data_full_augmentation/images'\ntrain_df = pd.read_csv('../input/fgvc8aug/data.csv', )","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:00:45.031583Z","iopub.execute_input":"2021-11-29T06:00:45.031814Z","iopub.status.idle":"2021-11-29T06:00:45.105871Z","shell.execute_reply.started":"2021-11-29T06:00:45.031789Z","shell.execute_reply":"2021-11-29T06:00:45.105088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess Dataset","metadata":{}},{"cell_type":"code","source":"train_df = train_df[[\"image\", \"labels\"]]\nmlb = MultiLabelBinarizer().fit(train_df.labels.apply(lambda x : x.split()))\nlabels = pd.DataFrame(mlb.transform(train_df.labels.apply(lambda x : x.split())), columns = mlb.classes_)\n\nlabels = pd.concat([train_df['image'], labels], axis=1)\nlabels.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:00:45.106987Z","iopub.execute_input":"2021-11-29T06:00:45.107319Z","iopub.status.idle":"2021-11-29T06:00:45.22957Z","shell.execute_reply.started":"2021-11-29T06:00:45.107293Z","shell.execute_reply":"2021-11-29T06:00:45.228959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def format_path(st):\n    return TRAIN_PATH + st\n\ntrain_paths = labels.image.apply(format_path).values\n\ntrain_labels = np.float32(labels.loc[:, 'complex':'scab'].values)\ntrain_paths, valid_paths, train_labels, valid_labels =\\\ntrain_test_split(train_paths, train_labels, test_size=0.15, random_state=2020)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:00:46.769557Z","iopub.execute_input":"2021-11-29T06:00:46.769952Z","iopub.status.idle":"2021-11-29T06:00:46.801496Z","shell.execute_reply.started":"2021-11-29T06:00:46.769922Z","shell.execute_reply":"2021-11-29T06:00:46.800917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_img(filepath,label):\n    image = tf.io.read_file(filepath)\n    image = tf.image.decode_jpeg(image, channels=CHANNELS)\n    image = tf.image.convert_image_dtype(image, tf.float32) \n    image = tf.image.resize(image, [HEIGHT,WIDTH])\n    return image,label","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:00:49.89375Z","iopub.execute_input":"2021-11-29T06:00:49.894167Z","iopub.status.idle":"2021-11-29T06:00:49.8988Z","shell.execute_reply.started":"2021-11-29T06:00:49.894138Z","shell.execute_reply":"2021-11-29T06:00:49.89825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((train_paths, train_labels))\n    .map(process_img, num_parallel_calls=AUTO)\n    .repeat()\n    .shuffle(512)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((valid_paths, valid_labels))\n    .map(process_img, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:00:52.394515Z","iopub.execute_input":"2021-11-29T06:00:52.394964Z","iopub.status.idle":"2021-11-29T06:00:52.530064Z","shell.execute_reply.started":"2021-11-29T06:00:52.394935Z","shell.execute_reply":"2021-11-29T06:00:52.529085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training model","metadata":{}},{"cell_type":"code","source":"with strategy.scope():  \n    base_model = EfficientNetB5(include_top = False, weights = 'imagenet')\n    base_model.trainabe = True\n\n    inputs = Input((HEIGHT, WIDTH, 3))\n    x = base_model(inputs, training = True)\n    x = GlobalAveragePooling2D()(x)\n    x = Dropout(top_dropout_rate)(x)\n    outputs = Dense(CLASSES, activation='sigmoid')(x)\n    \n    model = Model(inputs, outputs)\n    model.compile(optimizer ='adam' , loss='binary_crossentropy', metrics=['accuracy'])\n    model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:00:55.914394Z","iopub.execute_input":"2021-11-29T06:00:55.915021Z","iopub.status.idle":"2021-11-29T06:01:25.760117Z","shell.execute_reply.started":"2021-11-29T06:00:55.914983Z","shell.execute_reply":"2021-11-29T06:01:25.759239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = ModelCheckpoint(\n    final_model,\n    monitor = 'val_accuracy',\n    mode = 'max',\n    save_best_only = True,\n    save_weights_only= False ,\n    perior = 1,\n    verbose = 1\n)\n\nearly_stopping = EarlyStopping(\n    monitor = 'val_accuracy',\n    mode = 'auto',\n    min_delta = 0.0001,\n    patience = 5,\n    baseline = None,\n    restore_best_weights = True,\n    verbose = 1\n)\ndef build_lrfn(lr_start=0.00001, lr_max=0.00005, \n               lr_min=0.00001, lr_rampup_epochs=5, \n               lr_sustain_epochs=0, lr_exp_decay=.8):\n    lr_max = lr_max * strategy.num_replicas_in_sync\n\n    def lrfn(epoch):\n        if epoch < lr_rampup_epochs:\n            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min\n        return lr\n    return lrfn","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:01:25.761639Z","iopub.execute_input":"2021-11-29T06:01:25.76186Z","iopub.status.idle":"2021-11-29T06:01:25.770483Z","shell.execute_reply.started":"2021-11-29T06:01:25.761835Z","shell.execute_reply":"2021-11-29T06:01:25.769622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lrfn = build_lrfn()\nSTEPS_PER_EPOCH = train_labels.shape[0] // BATCH_SIZE\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:01:25.771609Z","iopub.execute_input":"2021-11-29T06:01:25.771841Z","iopub.status.idle":"2021-11-29T06:01:25.783227Z","shell.execute_reply.started":"2021-11-29T06:01:25.771816Z","shell.execute_reply":"2021-11-29T06:01:25.782288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = model.fit(\n    train_dataset, \n    validation_data = valid_dataset, \n    epochs = EPOCHS,\n    steps_per_epoch = STEPS_PER_EPOCH,\n    callbacks = [lr_schedule, early_stopping, checkpoint, CSVLogger(hist_path)]\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-29T06:01:25.784808Z","iopub.execute_input":"2021-11-29T06:01:25.785122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot history","metadata":{}},{"cell_type":"code","source":"def plot_hist(path):\n  history = pd.read_csv(path)\n\n  acc = history['accuracy']\n  val_acc = history['val_accuracy']\n\n  loss = history['loss']\n  val_loss = history['val_loss']\n  plt.style.use('fivethirtyeight')\n  plt.figure(figsize=(20, 10))\n\n  plt.subplot(1, 2, 1)\n  plt.plot(acc, label='Training Accuracy')\n  plt.plot(val_acc, label='Validation Accuracy')\n  plt.legend(loc='lower right')\n  plt.ylabel('Accuracy')\n  plt.ylim([min(plt.ylim()), 1])\n  plt.title('Training and Validation Accuracy')\n  plt.xlabel('epoch')\n\n  plt.subplot(1, 2, 2)\n  plt.plot(loss, label='Training Loss')\n  plt.plot(val_loss, label='Validation Loss')\n  plt.legend(loc='upper right')\n  plt.ylabel('Categorical Crossentropy')\n  plt.ylim([min(plt.ylim()), max(plt.ylim())])\n  plt.title('Training and Validation Loss')\n\n  plt.xlabel('epoch')\n  plt.savefig('evaluation.jpg')\n  plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T12:00:26.785337Z","iopub.execute_input":"2021-11-22T12:00:26.785675Z","iopub.status.idle":"2021-11-22T12:00:26.797714Z","shell.execute_reply.started":"2021-11-22T12:00:26.785627Z","shell.execute_reply":"2021-11-22T12:00:26.796495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(\"./EfficientNet_B5_Fullaug_Hist.log\")","metadata":{"execution":{"iopub.status.busy":"2021-11-22T12:00:30.700579Z","iopub.execute_input":"2021-11-22T12:00:30.701206Z","iopub.status.idle":"2021-11-22T12:00:31.391816Z","shell.execute_reply.started":"2021-11-22T12:00:30.701168Z","shell.execute_reply":"2021-11-22T12:00:31.391082Z"},"trusted":true},"execution_count":null,"outputs":[]}]}