{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport keras\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import to_categorical\nimport shutil\nfrom glob import glob\nimport pandas_profiling as pp\nimport cv2\nfrom google.cloud import storage\nfrom kaggle_datasets import KaggleDatasets\nfrom random import seed, randint, random, choice\nfrom PIL import Image\nimport tensorflow_addons as tfa\nimport sys\nfrom tensorflow.keras.optimizers import RMSprop, Adam, SGD\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom keras.callbacks import LearningRateScheduler\nimport gc\nimport math\nimport argparse\nimport warnings\nimport collections\nimport tensorflow.keras.backend as K\n\nwarnings.filterwarnings(\"ignore\")\nprint(\"Tensorflow version \" + tf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy\n\n\ndef build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n        \n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True):\n    def augment(img):\n\n        img = tf.image.random_crop(img, [im_size, im_size, 3])\n\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        k90 = randint(0, 3)\n        img = tf.image.rot90(img, k=k90)\n        \n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n    \n    return dset","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 12\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('plant-pathology-2021-fgvc8')\nprint('BATCH SIZE:', BATCH_SIZE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def swish(x):\n    return x * tf.nn.sigmoid(x)\n\ndef SEBlock(input_filters,se_output_filters,se_ratio=0.25):\n    def block(inputs):\n        num_reduced_filters = max(1, int(input_filters * se_ratio))\n        x = inputs\n        x = tf.keras.layers.Lambda(lambda a: K.mean(a, axis=[1, 2], keepdims=True))(x)\n        x = tf.keras.layers.Conv2D(num_reduced_filters,kernel_size=(1, 1),padding='same')(x)\n        x = swish(x)\n        x = tf.keras.layers.Conv2D(se_output_filters,kernel_size=(1, 1),padding='same',)(x)\n        x = tf.keras.layers.Activation('sigmoid')(x)\n        out = tf.keras.layers.Multiply()([x, inputs])\n        return out\n    return block\n\nclass DropConnect(tf.keras.layers.Layer):\n    def __init__(self, drop_connect_rate=0.):\n        super().__init__()\n        self.drop_connect_rate = drop_connect_rate\n\n    def call(self, inputs, training=None):\n        def drop_connect():\n            survival_prob = 1.0 - self.drop_connect_rate\n            batch_size = tf.shape(inputs)[0]\n            random_tensor = survival_prob\n            random_tensor += tf.random_uniform([batch_size, 1, 1, 1], dtype=inputs.dtype)\n            binary_tensor = tf.floor(random_tensor)\n            output = tf.div(inputs, survival_prob) * binary_tensor\n            return output\n        return K.in_train_phase(drop_connect, inputs, training=training)\n\ndef MBConvBlock(input_filters, output_filters,kernel_size, strides,expand_ratio,drop_connect_rate):\n    def block(inputs):\n        se_output_filters = input_filters * expand_ratio\n        if expand_ratio != 1:\n            x = tf.keras.layers.Conv2D(se_output_filters,kernel_size=(1, 1),padding='same',use_bias=False)(inputs)\n            x = tf.keras.layers.BatchNormalization()(x)\n            x = swish(x)\n        else:\n            x = inputs\n        x = tf.keras.layers.DepthwiseConv2D((kernel_size, kernel_size),strides=strides,padding='same',use_bias=False)(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = swish(x)\n        x = SEBlock(input_filters,se_output_filters)(x)\n        x = tf.keras.layers.Conv2D(output_filters,kernel_size=(1, 1),padding='same',use_bias=False)(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n        if all(s == 1 for s in strides) and (input_filters == output_filters):\n            if drop_connect_rate:\n                x = DropConnect(drop_connect_rate)(x)\n\n            x = tf.keras.layers.Add()([x, inputs])\n        return x\n    return block\n\n\nBlockArgs = collections.namedtuple('BlockArgs', ['kernel_size', 'num_repeat', 'input_filters', 'output_filters','expand_ratio', 'strides'])\nblock_args_list = [\n    BlockArgs(kernel_size=3, num_repeat=1, input_filters=32, output_filters=16,expand_ratio=1, strides=[1, 1]),\n    BlockArgs(kernel_size=3, num_repeat=2, input_filters=16, output_filters=24,expand_ratio=6, strides=[2, 2]),\n    BlockArgs(kernel_size=5, num_repeat=2, input_filters=24, output_filters=40,expand_ratio=6, strides=[2, 2]),\n    BlockArgs(kernel_size=3, num_repeat=3, input_filters=40, output_filters=80,expand_ratio=6, strides=[2, 2]),\n    BlockArgs(kernel_size=5, num_repeat=3, input_filters=80, output_filters=112,expand_ratio=6, strides=[1, 1]),\n    BlockArgs(kernel_size=5, num_repeat=4, input_filters=112, output_filters=192,expand_ratio=6, strides=[2, 2]),\n    BlockArgs(kernel_size=3, num_repeat=1, input_filters=192, output_filters=320,expand_ratio=6, strides=[1, 1])\n]\nstride_count = 5\nnum_blocks = 16\n\ndef EfficientNet(input_shape,classes,width_coefficient: float,depth_coefficient: float,include_top=True,dropout_rate=0.,drop_connect_rate=0.): \n    inputs = tf.keras.layers.Input(shape=input_shape)\n    x = inputs\n    x = tf.keras.layers.Conv2D(filters=int(32*width_coefficient), kernel_size=(3,3),strides=(2,2),padding='same',use_bias=False)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = swish(x)    \n    drop_connect_rate_per_block = drop_connect_rate / float(num_blocks)\n    for block_idx, block_args in enumerate(block_args_list):\n        my_input_filters = int(block_args.input_filters*width_coefficient)\n        my_output_filters = int(block_args.output_filters*width_coefficient)\n        my_num_repeat = math.ceil(block_args.num_repeat*depth_coefficient)\n        x = MBConvBlock(my_input_filters, my_output_filters,block_args.kernel_size, block_args.strides,block_args.expand_ratio,drop_connect_rate_per_block * block_idx)(x)\n        if my_num_repeat > 1:\n            my_input_filters = my_output_filters\n            my_strides = [1, 1]\n        for _ in range(my_num_repeat - 1):\n            x = MBConvBlock(my_input_filters, my_output_filters,block_args.kernel_size, my_strides,block_args.expand_ratio,drop_connect_rate_per_block * block_idx)(x)\n    x = tf.keras.layers.Conv2D(filters=int(1280*width_coefficient),kernel_size=(1, 1),padding='same',use_bias=False)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = swish(x)\n    if include_top:\n        x = tf.keras.layers.GlobalAveragePooling2D()(x)\n        if dropout_rate > 0:\n            x = tf.keras.layers.Dropout(dropout_rate)(x)\n        x = tf.keras.layers.Dense(classes)(x)\n        x = tf.keras.layers.Activation('softmax')(x)\n    outputs = x\n    model = tf.keras.models.Model(inputs, outputs)\n    return model\n\ndef EfficientNetB0(input_shape,classes,include_top=True,dropout_rate=0.4,drop_connect_rate=0.): \n    return  EfficientNet(input_shape,classes,1.0,1.0,include_top=include_top,dropout_rate=dropout_rate,drop_connect_rate=drop_connect_rate)\n\ndef EfficientNetB1(input_shape,classes,include_top=True,dropout_rate=0.2,drop_connect_rate=0.): \n    return  EfficientNet(input_shape,classes,1.0,1.1,include_top=include_top,dropout_rate=dropout_rate,drop_connect_rate=drop_connect_rate)\n\ndef EfficientNetB2(input_shape,classes,include_top=True,dropout_rate=0.3,drop_connect_rate=0.): \n    return  EfficientNet(input_shape,classes,1.1,1.2,include_top=include_top,dropout_rate=dropout_rate,drop_connect_rate=drop_connect_rate)\n\ndef EfficientNetB3(input_shape,classes,include_top=True,dropout_rate=0.3,drop_connect_rate=0.): \n    return  EfficientNet(input_shape,classes,1.2,1.4,include_top=include_top,dropout_rate=dropout_rate,drop_connect_rate=drop_connect_rate)\n\ndef EfficientNetB4(input_shape,classes,include_top=True,dropout_rate=0.4,drop_connect_rate=0.): \n    return  EfficientNet(input_shape,classes,1.4,1.8,include_top=include_top,dropout_rate=dropout_rate,drop_connect_rate=drop_connect_rate)\n\ndef EfficientNetB5(input_shape,classes,include_top=True,dropout_rate=0.4,drop_connect_rate=0.): \n    return  EfficientNet(input_shape,classes,1.6,2.2,include_top=include_top,dropout_rate=dropout_rate,drop_connect_rate=drop_connect_rate)\n\ndef EfficientNetB6(input_shape,classes,include_top=True,dropout_rate=0.5,drop_connect_rate=0.): \n    return  EfficientNet(input_shape,classes,1.8,2.6,include_top=include_top,dropout_rate=dropout_rate,drop_connect_rate=drop_connect_rate)\n\ndef EfficientNetB7(input_shape,classes,include_top=True,dropout_rate=0.5,drop_connect_rate=0.): \n    return  EfficientNet(input_shape,classes,2.0,3.1,include_top=include_top,dropout_rate=dropout_rate,drop_connect_rate=drop_connect_rate)\n\ndef EfficientNetB8(input_shape,classes,include_top=True,dropout_rate=0.5,drop_connect_rate=0.): \n    return  EfficientNet(input_shape,classes,2.2,3.6,include_top=include_top,dropout_rate=dropout_rate,drop_connect_rate=drop_connect_rate)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMSIZES = (224, 240, 260, 300, 380, 456, 528, 600, 672, 800)\nim_size = IMSIZES[8]\n\nzoom_ratio = 1.2\nim_size_zoom = int(im_size * zoom_ratio)\nprint(im_size, im_size_zoom)\n\ndf = pd.read_csv('/kaggle/input/plant-pathology-2021-fgvc8/train.csv')    \n# df = df.iloc[::-1]\n# df = df.reset_index(drop=True)\n\nclass_name = df.labels.unique().tolist()\nclass_name.sort()\nclass_num = len(class_name)\n\nn_labels = 5\n\nprint(class_name) \nprint(class_num)\n\nimg_per_class = []\nfor c in class_name:\n    img_per_class.append(df[df.labels==c])\n\none_hot = {'healthy':                  [0, 0, 0, 0, 0],\n           'complex':                  [1, 0, 0, 0, 0],\n           'scab':                     [0, 1, 0, 0, 0],\n           'frog_eye_leaf_spot':       [0, 0, 1, 0, 0],\n           'rust':                     [0, 0, 0, 1, 0],\n           'powdery_mildew':           [0, 0, 0, 0, 1],\n           'rust frog_eye_leaf_spot':         [0, 0, 1, 1, 0],\n           'scab frog_eye_leaf_spot':         [0, 1, 1, 0, 0],\n           'frog_eye_leaf_spot complex':      [1, 0, 1, 0, 0],\n           'scab frog_eye_leaf_spot complex': [1, 1, 1, 0, 0],\n           'powdery_mildew complex':          [1, 0, 0, 0, 1],\n           'rust complex':                    [1, 0, 0, 1, 0]\n         }\n    \ndf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loop = [1] * class_num\n\ntrain_paths = []\nvalid_paths = []\ntrain_labels = []\nvalid_labels = []\n\nTESTSIZE = 0.1\nLIMIT = 5000\n\nfor i in range(class_num):\n    fullsize = img_per_class[i].shape[0]\n    print(str(class_name[i]) + ': ' + str(fullsize))\n    fullsize = min(fullsize, LIMIT)\n    size = int(fullsize * TESTSIZE)\n    cnt = 0\n    for j in range(loop[i]):\n        valid_paths += list(GCS_DS_PATH + '/train_images/' + img_per_class[i][:size]['image'])\n        train_paths += list(GCS_DS_PATH + '/train_images/' + img_per_class[i][size:fullsize]['image'])\n        valid_labels += list(img_per_class[i][:size]['labels'])\n        train_labels += list(img_per_class[i][size:fullsize]['labels'])\n        cnt += len(list(GCS_DS_PATH + '/train_images/' + img_per_class[i][size:fullsize]['image']))\n    print(str(class_name[i]) + '_train: ' + str(cnt))\n    \nvalid_labels = [one_hot[x] for x in valid_labels]\ntrain_labels = [one_hot[x] for x in train_labels]\n\nprint()\nprint('Train: ', len(train_paths))\nprint('Valid: ', len(valid_paths))\nprint('Sum:', len(train_paths) + len(valid_paths))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_decoder = build_decoder(with_labels=True, target_size=(im_size_zoom, im_size_zoom))\nvalid_decoder = build_decoder(with_labels=True, target_size=(im_size, im_size))\n\ntrain_dataset = build_dataset(\n    train_paths, train_labels, bsize=BATCH_SIZE, decode_fn=train_decoder\n)\n\nvalid_dataset = build_dataset(\n    valid_paths, valid_labels, bsize=BATCH_SIZE, decode_fn=valid_decoder,\n    repeat=False, shuffle=False, augment=False\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clean up\n\ndel df\ndel img_per_class\ndel im_size_zoom\ndel class_num\ndel class_name\n\ndel train_decoder\ndel valid_decoder\n\ndel loop\n\ndel valid_paths\ndel train_labels\ndel valid_labels\ndel one_hot\n\ndel TESTSIZE\ndel LIMIT\n\ndel i\ndel j\ndel c\ndel fullsize\n\ndel GCS_DS_PATH\n\ngc.collect","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.optimizers import RMSprop, Adam, SGD\nfrom keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, Input, GlobalAveragePooling2D, ReLU, Flatten, Dense, Dropout, BatchNormalization, MaxPooling2D, GlobalMaxPooling2D\nimport tensorflow_addons\n\nwith strategy.scope():\n    \n    base = EfficientNetB8(include_top=False, input_shape=(im_size, im_size, 3), classes=n_labels)\n    \n    model = Sequential()\n    model.add(base)\n    model.add(GlobalAveragePooling2D())\n    model.add(Dense(n_labels, activation='sigmoid'))\n    \n    model.compile(loss='binary_crossentropy', optimizer=Adam(lr=5e-5), \n                  metrics= ['binary_accuracy'])\n    \n    model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights('../input/testeffnet/besteff8.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint = ModelCheckpoint('besteff8.h5',\n                             save_best_only=True,\n                             save_weights_only=True,\n                             monitor='val_loss',\n                             mode='auto',\n                             verbose=1)\n\nreduce_lr = ReduceLROnPlateau(monitor='val_loss',\n                              mode='auto',\n                              factor=0.2,\n                              patience=20,\n                              min_lr=1e-6,\n                              verbose=1)\n\n# es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=15)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"steps_per_epoch = len(train_paths) // BATCH_SIZE\n\ndel train_paths\ndel base\ndel strategy\ngc.collect\n\nEPOCHS = 70","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"his = model.fit(\n            train_dataset, \n            epochs=EPOCHS,\n            verbose=1,\n            callbacks=[checkpoint, reduce_lr],\n            steps_per_epoch=steps_per_epoch,\n            validation_data=valid_dataset)\n\nmodel.save_weights('eff8net.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.read_csv('/kaggle/input/plant-pathology-2021-fgvc8/train.csv')\n# df[-50:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/plant-pathology-2021-fgvc8/train.csv')\n\none_hot = {'healthy':            [0, 0, 0, 0, 0],\n           'complex':            [1, 0, 0, 0, 0],\n           'scab':               [0, 1, 0, 0, 0],\n           'frog_eye_leaf_spot': [0, 0, 1, 0, 0],\n           'rust':               [0, 0, 0, 1, 0],\n           'powdery_mildew':     [0, 0, 0, 0, 1],\n           'scab frog_eye_leaf_spot': [0, 1, 1, 0, 0],\n           'rust frog_eye_leaf_spot': [0, 0, 1, 1, 0],\n           'frog_eye_leaf_spot complex': [1, 0, 1, 0, 0],\n           'scab frog_eye_leaf_spot complex': [1, 1, 1, 0, 0],\n           'powdery_mildew complex': [1, 0, 0, 0, 1],\n           'rust complex': [1, 0, 0, 1, 0]}\n\nx = 'fffe105cf6808292.jpg'\nimg = cv2.imread(f'/kaggle/input/plant-pathology-2021-fgvc8/train_images/{x}')\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nimg = cv2.resize(img, (im_size, im_size))\nplt.imshow(img)\nimg = np.array(img)\nimg = np.reshape(img, (1, im_size, im_size, 3))\nprint(df[df.image==x].labels.values[0])\nprint(one_hot[df[df.image==x].labels.values[0]])\nprint(model.predict(img / 255.0))\n\ndel one_hot\ndel x\ndel img\ndel df\n\ngc.collect","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}