{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!nvidia-smi ","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:51:47.663149Z","iopub.execute_input":"2021-06-07T11:51:47.663521Z","iopub.status.idle":"2021-06-07T11:51:48.340188Z","shell.execute_reply.started":"2021-06-07T11:51:47.663441Z","shell.execute_reply":"2021-06-07T11:51:48.339309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers.experimental import preprocessing\nfrom tensorflow.keras.applications import ResNet50, EfficientNetB0, EfficientNetB1, EfficientNetB2, EfficientNetB3, EfficientNetB4, EfficientNetB7, VGG16, ResNet50V2, InceptionV3\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential\nimport pandas as pd\nimport imgaug as ia\nimport imgaug.augmenters as iaa\nimport imageio\nimport glob, os\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nimport logging\nimport tensorflow_addons as tfa\n\ntf.config.list_physical_devices('GPU')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:51:48.3434Z","iopub.execute_input":"2021-06-07T11:51:48.34367Z","iopub.status.idle":"2021-06-07T11:51:55.413086Z","shell.execute_reply.started":"2021-06-07T11:51:48.343643Z","shell.execute_reply":"2021-06-07T11:51:55.412314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# enable XLA and mixed precision\nfrom tensorflow.keras import mixed_precision\npolicy = mixed_precision.Policy('mixed_float16')\nmixed_precision.set_global_policy(policy)\ntf.config.optimizer.set_jit(True)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:51:55.416198Z","iopub.execute_input":"2021-06-07T11:51:55.416451Z","iopub.status.idle":"2021-06-07T11:51:55.425858Z","shell.execute_reply.started":"2021-06-07T11:51:55.416426Z","shell.execute_reply":"2021-06-07T11:51:55.425076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n    'model': 'EfficientNetB4',\n    'img_shape': (380, 380),\n    'max_epochs': 50,\n    'max_epochs_per_fit': 10,\n    'num_classes': 5,\n    'batch_size': 128,\n    'transfer_learning': True,\n    'pre_training_epochs': 10,\n    'pre_training_only': False,\n    'pre_training_learning_rate': 5e-2,\n    'pre_training_min_delta': 0.01,\n    'pre_training_patience': 3,\n    'fine_tuning_learning_rate': 1e-4,\n    'fine_tuning_min_delta': 0.0,\n    'fine_tuning_patience': 0,\n    'fine_tuning_unfreeze_interval': 15,\n    'f1_average': 'macro',\n    'f1_threshold': 0.5,\n    'validation_split': 0.15,\n    'csv_location': '../input/plant-pathology-fgvc78-640px/train.csv',\n    'data_location': '../input/plant-pathology-fgvc78-640px/train_images',\n    'gamma': 0.15 # for label smoothing\n}\nmodel_map = {\n    'EfficientNetB0': EfficientNetB0,\n    'EfficientNetB1': EfficientNetB1,\n    'EfficientNetB2': EfficientNetB2,\n    'EfficientNetB3': EfficientNetB3,\n    'EfficientNetB4': EfficientNetB4,\n    'ResNet50': ResNet50,\n    'ResNet50V2': ResNet50V2,\n    'InceptionV3': InceptionV3\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:51:55.429314Z","iopub.execute_input":"2021-06-07T11:51:55.429557Z","iopub.status.idle":"2021-06-07T11:51:55.438027Z","shell.execute_reply.started":"2021-06-07T11:51:55.429533Z","shell.execute_reply":"2021-06-07T11:51:55.437147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# labels need to be converted to a list of labels (space separator)\ndf = pd.read_csv(config['csv_location'], delimiter=',')\nmake_multi_label = lambda x: x.split(' ')\ndf.labels = df.labels.apply(make_multi_label)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:51:55.441343Z","iopub.execute_input":"2021-06-07T11:51:55.441586Z","iopub.status.idle":"2021-06-07T11:51:55.500481Z","shell.execute_reply.started":"2021-06-07T11:51:55.441562Z","shell.execute_reply":"2021-06-07T11:51:55.499755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# EXPERIMENTAL: remove healthy label\ndf.labels = df.labels.apply(lambda x: [item for item in x if item != 'healthy'])\ndf","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:51:55.501634Z","iopub.execute_input":"2021-06-07T11:51:55.501952Z","iopub.status.idle":"2021-06-07T11:51:55.53648Z","shell.execute_reply.started":"2021-06-07T11:51:55.501918Z","shell.execute_reply":"2021-06-07T11:51:55.535789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check if dataframe contains multiple labels\ndf","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:51:55.537664Z","iopub.execute_input":"2021-06-07T11:51:55.53805Z","iopub.status.idle":"2021-06-07T11:51:55.551204Z","shell.execute_reply.started":"2021-06-07T11:51:55.538011Z","shell.execute_reply":"2021-06-07T11:51:55.550109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##augmentation test \n#class AugmenterConfig:\n#    name: str = 'basic'\n#    extension: str = 'default'\n#    width: int = 224\n#    height: int = 224\n#    pad_mode: str = 'edge'\n#    position: str = 'center'\n#        \n#def basic_augmenter(cfg: AugmenterConfig):\n#    return iaa.Sequential([\n#        iaa.Resize({'shorter-side': 'keep-aspect-ratio', 'longer-side': max(cfg.width, cfg.height)}),\n#       # iaa.CropToSquare(position='center')\n#        iaa.PadToFixedSize(width=cfg.width, height=cfg.height, position=cfg.position, pad_mode=cfg.pad_mode, pad_cval=(0, 0))\n#    ])\n#\n#cfg = AugmenterConfig()\n#%matplotlib inline \n#aug = basic_augmenter(cfg)\n#path='../input/plant-pathology-2021-fgvc8/train_images/803553d5df59cac2.jpg'\n#img = imageio.imread(path)\n#new_img = aug(images=[img])[0]\n#plt.imshow(new_img)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-07T11:47:40.044548Z","iopub.execute_input":"2021-06-07T11:47:40.044951Z","iopub.status.idle":"2021-06-07T11:47:40.051543Z","shell.execute_reply.started":"2021-06-07T11:47:40.044883Z","shell.execute_reply":"2021-06-07T11:47:40.05044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## images need to be cropped offline to speed up training by factor of ~50\n#cfg = AugmenterConfig()\n#aug = basic_augmenter(cfg)\n#src_path = Path('../input/plant-pathology-2021-fgvc8/train_images')\n#dst_path = Path('./train_images')\n#dst_path.mkdir(exist_ok=True)\n#for file in glob.glob(f'{src_path}/*.jpg'):\n#    new_img = aug(images=[img])[0]\n#    pf = Path(file)\n#    imageio.imwrite(dst_path / f'{pf.stem}{pf.suffix}', new_img)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:47:40.057573Z","iopub.execute_input":"2021-06-07T11:47:40.057838Z","iopub.status.idle":"2021-06-07T11:47:40.064561Z","shell.execute_reply.started":"2021-06-07T11:47:40.057814Z","shell.execute_reply":"2021-06-07T11:47:40.063808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ndf_train, df_val = train_test_split(df, test_size=config['validation_split'])","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:52:10.576615Z","iopub.execute_input":"2021-06-07T11:52:10.576938Z","iopub.status.idle":"2021-06-07T11:52:10.741984Z","shell.execute_reply.started":"2021-06-07T11:52:10.576907Z","shell.execute_reply":"2021-06-07T11:52:10.741185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = config['data_location']\ntrain_datagen = ImageDataGenerator()  # rescale=1. / 255)\ntrain_generator = train_datagen.flow_from_dataframe(df_train, directory=data_dir, x_col='image', y_col='labels', weight_col=None, target_size=config['img_shape'], color_mode='rgb', classes=None, class_mode='categorical', batch_size=128, shuffle=True, seed=None, save_to_dir=None, save_prefix='', save_format='jpg', subset=None, interpolation='nearest', validate_filenames=True)\nval_datagen = ImageDataGenerator()  # rescale=1. / 255)\nval_generator = train_datagen.flow_from_dataframe(df_val, directory=data_dir, x_col='image', y_col='labels', weight_col=None, target_size=config['img_shape'], color_mode='rgb', classes=None, class_mode='categorical', batch_size=128, shuffle=True, seed=None, save_to_dir=None, save_prefix='', save_format='jpg', subset=None, interpolation='nearest', validate_filenames=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:52:10.852873Z","iopub.execute_input":"2021-06-07T11:52:10.853269Z","iopub.status.idle":"2021-06-07T11:52:47.135268Z","shell.execute_reply.started":"2021-06-07T11:52:10.853234Z","shell.execute_reply":"2021-06-07T11:52:47.134328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator.class_indices","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:52:47.13669Z","iopub.execute_input":"2021-06-07T11:52:47.137215Z","iopub.status.idle":"2021-06-07T11:52:47.143227Z","shell.execute_reply.started":"2021-06-07T11:52:47.137175Z","shell.execute_reply":"2021-06-07T11:52:47.142201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_augmentation = Sequential(\n    [\n        preprocessing.RandomRotation(factor=0.15),\n        #preprocessing.RandomTranslation(height_factor=0.1, width_factor=0.1),\n        # preprocessing.RandomFlip(),\n        preprocessing.RandomContrast(factor=0.1),\n    ],\n    name=\"img_augmentation\",\n)\n\ndef unfreeze_model(model, to_unfreeze=20, lr=1e-4):\n    # We unfreeze the top 20 layers while leaving BatchNorm layers frozen\n    for layer in model.layers[-to_unfreeze:]:\n        if not isinstance(layer, layers.BatchNormalization):\n            layer.trainable = True\n\n    optimizer = tf.keras.optimizers.Adam(learning_rate=lr)\n    f1 = tfa.metrics.F1Score(num_classes=config['num_classes'], threshold=config['f1_threshold'], average=config['f1_average'])\n    loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, label_smoothing=config['gamma'], reduction=\"auto\", name=\"binary_crossentropy\")\n    model.compile(\n        optimizer=optimizer, loss=loss, metrics=[\"accuracy\", f1]\n    )\n    return model\n\ndef generic_builder(name, net, lr=1e-2, dropout_rate=0.2, num_classes=6, img_shape=(380,380,3)):\n    inputs = layers.Input(img_shape)\n    x = img_augmentation(inputs)\n    model = net(include_top=False, input_tensor=x, weights='imagenet')\n    # Freeze the pretrained weights\n    model.trainable = False\n\n    # Rebuild top\n    x = layers.GlobalAveragePooling2D(name=\"avg_pool\")(model.output)\n    x = layers.BatchNormalization()(x)\n    top_dropout_rate = dropout_rate\n    x = layers.Dropout(top_dropout_rate, name=\"top_dropout\")(x)\n    x = layers.Dense(\n        num_classes, activation=\"sigmoid\", name=\"pred\")(x)\n    outputs = tf.cast(x, tf.float32) # workaround for fp16 crossentropy bug\n    \n    # Compile\n    model = tf.keras.Model(inputs, outputs, name=name)\n    optimizer = tf.keras.optimizers.Adam(learning_rate=lr)\n    f1 = tfa.metrics.F1Score(num_classes=config['num_classes'], threshold=config['f1_threshold'], average=config['f1_average'])\n    loss = tf.keras.losses.BinaryCrossentropy(from_logits=True, label_smoothing=config['gamma'], reduction=\"auto\", name=\"binary_crossentropy\")\n    model.compile(\n        optimizer=optimizer, loss=loss, metrics=[\"accuracy\", f1]\n    )\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:48:04.682895Z","iopub.execute_input":"2021-06-07T11:48:04.683324Z","iopub.status.idle":"2021-06-07T11:48:05.57301Z","shell.execute_reply.started":"2021-06-07T11:48:04.683285Z","shell.execute_reply":"2021-06-07T11:48:05.572161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = generic_builder(config['model'], model_map[config['model']], num_classes=config['num_classes'], img_shape=(config['img_shape'][0], config['img_shape'][1], 3))","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:48:05.574336Z","iopub.execute_input":"2021-06-07T11:48:05.57468Z","iopub.status.idle":"2021-06-07T11:48:09.096995Z","shell.execute_reply.started":"2021-06-07T11:48:05.574643Z","shell.execute_reply":"2021-06-07T11:48:09.096151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.keras.callbacks import EarlyStopping\npre_training_early_stopping_callback = EarlyStopping(\n    monitor='val_f1_score', mode='max', min_delta=config['pre_training_min_delta'], patience=config['pre_training_patience'], restore_best_weights=True)\npre_training_callbacks = [pre_training_early_stopping_callback]\n\nfine_tuning_early_stopping_callback = EarlyStopping(\n    monitor='val_f1_score', mode='max', min_delta=config['fine_tuning_min_delta'], patience=config['fine_tuning_patience'], restore_best_weights=True)\nfine_tuning_callbacks = [fine_tuning_early_stopping_callback]","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:48:09.098349Z","iopub.execute_input":"2021-06-07T11:48:09.09869Z","iopub.status.idle":"2021-06-07T11:48:09.104636Z","shell.execute_reply.started":"2021-06-07T11:48:09.098653Z","shell.execute_reply":"2021-06-07T11:48:09.103627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs_to_date = 0\nhistory = model.fit(train_generator, validation_data=val_generator, epochs=config['pre_training_epochs'], callbacks=pre_training_callbacks)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:48:09.106008Z","iopub.execute_input":"2021-06-07T11:48:09.106528Z","iopub.status.idle":"2021-06-07T11:49:59.93788Z","shell.execute_reply.started":"2021-06-07T11:48:09.106489Z","shell.execute_reply":"2021-06-07T11:49:59.935465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config['transfer_learning'] and not config['pre_training_only']:\n    for to_unfreeze in range(config['fine_tuning_unfreeze_interval'], len(model.layers), config['fine_tuning_unfreeze_interval']):\n        model = unfreeze_model(model, to_unfreeze, lr=config['fine_tuning_learning_rate'])\n        trainable_layers = len(\n            [1 for layer in model.layers if layer.trainable is True])\n        logging.info(f'Fine-tuning on {trainable_layers}')\n        epochs_to_date += len(history.history['loss'])\n        if epochs_to_date >= config['max_epochs']:\n            break\n        epochs_to_do = epochs_to_date+config['max_epochs_per_fit'] if epochs_to_date + \\\n            config['max_epochs_per_fit'] < config['max_epochs'] else config['max_epochs']\n        history = model.fit(train_generator, validation_data=val_generator, initial_epoch=epochs_to_date,\n                            epochs=epochs_to_do, callbacks=fine_tuning_callbacks)","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:49:59.938937Z","iopub.status.idle":"2021-06-07T11:49:59.939327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('model-best.h5')","metadata":{"execution":{"iopub.status.busy":"2021-06-07T11:49:59.94031Z","iopub.status.idle":"2021-06-07T11:49:59.94089Z"},"trusted":true},"execution_count":null,"outputs":[]}]}