{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#It's forked from https://www.kaggle.com/code/tchaye59/efficientnet-tensorflow-baseline-tpu\n#I have not yet been able to tune the parameters sufficiently, but I confirmed roughly 0.6 or better score with training 30 epochs!","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:54:45.644051Z","iopub.execute_input":"2022-04-12T12:54:45.644348Z","iopub.status.idle":"2022-04-12T12:54:46.414314Z","shell.execute_reply.started":"2022-04-12T12:54:45.64432Z","shell.execute_reply":"2022-04-12T12:54:46.413445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip uninstall -y tensorflow_datasets\n! pip install tensorflow_datasets==4.4.0\n!pip install -U keras-efficientnet-v2\n!pip install tensorflow_addons\n!pip install -U scikit-learn","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-12T12:54:46.416295Z","iopub.execute_input":"2022-04-12T12:54:46.41657Z","iopub.status.idle":"2022-04-12T12:55:22.519719Z","shell.execute_reply.started":"2022-04-12T12:54:46.416538Z","shell.execute_reply":"2022-04-12T12:55:22.518686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom sklearn.preprocessing import LabelEncoder\nimport dill\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf\nfrom tensorflow import keras \nfrom tensorflow.keras import backend as K \nimport seaborn as sns\nimport random\nimport gc\nfrom tqdm.notebook import tqdm\nimport tensorflow_addons as tfa\nimport sys\nsys.path.append(\"../input/sorghum100cultivarjpgtfrecords512x512\")\nfrom fgvc_dataset import FGVCDataset\nimport keras_efficientnet_v2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-12T12:55:22.521774Z","iopub.execute_input":"2022-04-12T12:55:22.524585Z","iopub.status.idle":"2022-04-12T12:55:22.534325Z","shell.execute_reply.started":"2022-04-12T12:55:22.524528Z","shell.execute_reply":"2022-04-12T12:55:22.533407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NEW on TPU in TensorFlow 24: shorter cross-compatible TPU/GPU/multi-GPU/cluster-GPU detection code\ntpu = None\ntry: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # detect GPUs\n    #strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\n    \n#strategy,tpu = tf.distribute.MirroredStrategy(devices=[\"TPU:0\", \"TPU:1\",\"TPU:2\"]),True\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)\n\n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-12T12:55:22.537122Z","iopub.execute_input":"2022-04-12T12:55:22.537784Z","iopub.status.idle":"2022-04-12T12:55:28.33126Z","shell.execute_reply.started":"2022-04-12T12:55:22.537736Z","shell.execute_reply":"2022-04-12T12:55:28.330222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# We prepared the dataset in the following notebooks:\n\nhttps://www.kaggle.com/tchaye59/images-to-jpeg-512x512\n\nhttps://www.kaggle.com/tchaye59/512x512-images-to-tfrecords","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/sorghum-id-fgvc-9/train_cultivar_mapping.csv\")\n#remove 3329\t.DS_Store\tNaN\ntrain_df = train_df[train_df.cultivar.notnull()]\n\ntest_df = pd.read_csv(\"../input/sorghum-id-fgvc-9/sample_submission.csv\")\nGCS_PATH = KaggleDatasets().get_gcs_path('sorghum100cultivarjpgtfrecords512x512')","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:55:28.332485Z","iopub.execute_input":"2022-04-12T12:55:28.332756Z","iopub.status.idle":"2022-04-12T12:55:28.708753Z","shell.execute_reply.started":"2022-04-12T12:55:28.332729Z","shell.execute_reply":"2022-04-12T12:55:28.707854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"USE_SAVED_MODEL = False","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:55:28.710496Z","iopub.execute_input":"2022-04-12T12:55:28.711157Z","iopub.status.idle":"2022-04-12T12:55:28.715873Z","shell.execute_reply.started":"2022-04-12T12:55:28.71111Z","shell.execute_reply":"2022-04-12T12:55:28.714976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load dataset","metadata":{}},{"cell_type":"code","source":"%%time\ndata_dir= GCS_PATH+\"/fgvc_dataset\"\nbuilder = FGVCDataset(data_dir=data_dir)\nbuilder.download_and_prepare()\ntrain_ds = builder.as_dataset()['train']\ntest_ds = builder.as_dataset()['test']","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-12T12:55:28.717437Z","iopub.execute_input":"2022-04-12T12:55:28.717929Z","iopub.status.idle":"2022-04-12T12:55:29.83787Z","shell.execute_reply.started":"2022-04-12T12:55:28.717887Z","shell.execute_reply":"2022-04-12T12:55:29.837029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image_batch(images: list):\n    \"\"\"\n    Displays a batch of image present in images\n    \"\"\"\n    fig = plt.figure(figsize=(10,5))\n    for idx in range(6):\n        ax = plt.subplot(2, 3, idx+1)\n        plt.imshow(images[idx])\n        plt.axis(\"off\")\n\ndef show_dataset(dataset):\n    batch = next(iter(dataset))\n    images, labels = batch\n    \n    plt.figure(figsize=(10, 10))\n    for idx in range(9):\n        ax = plt.subplot(3, 3, idx + 1)\n        plt.imshow(images[idx].numpy().astype(\"uint8\"))\n        plt.title(\"Class: {}\".format(labels[idx].numpy().decode()))\n        plt.axis(\"off\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-04-12T12:55:29.839285Z","iopub.execute_input":"2022-04-12T12:55:29.83953Z","iopub.status.idle":"2022-04-12T12:55:29.848553Z","shell.execute_reply.started":"2022-04-12T12:55:29.839484Z","shell.execute_reply":"2022-04-12T12:55:29.84766Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_dataset(train_ds.map(lambda data:(data['img'],data['cultivar'])).batch(9))","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:55:29.849694Z","iopub.execute_input":"2022-04-12T12:55:29.849913Z","iopub.status.idle":"2022-04-12T12:55:32.343128Z","shell.execute_reply.started":"2022-04-12T12:55:29.849888Z","shell.execute_reply":"2022-04-12T12:55:32.342414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le = dill.load(open(\"../input/sorghum100cultivarjpgtfrecords512x512/le.dill\",\"rb\"))\ntrain_df[\"target\"] = le.transform(train_df.cultivar)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:55:32.34543Z","iopub.execute_input":"2022-04-12T12:55:32.345789Z","iopub.status.idle":"2022-04-12T12:55:32.363727Z","shell.execute_reply.started":"2022-04-12T12:55:32.345761Z","shell.execute_reply":"2022-04-12T12:55:32.36295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 123\nHEIGHT = 512\nWIDTH = 512\nMAGNITUDE = 10\nN_CLASSES = len(train_df.cultivar.unique())\nBATCH_SIZE = 64 if tpu else 4\nIMG_DIR = \"../input/sorghum-id-fgvc-9\" if not tpu else GCS_PATH","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:55:32.365102Z","iopub.execute_input":"2022-04-12T12:55:32.365355Z","iopub.status.idle":"2022-04-12T12:55:32.372241Z","shell.execute_reply.started":"2022-04-12T12:55:32.365327Z","shell.execute_reply":"2022-04-12T12:55:32.371418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TF functions","metadata":{}},{"cell_type":"code","source":"# def data_augmentation():\n#     return keras.Sequential([\n#         #keras.layers.experimental.preprocessing.RandomZoom(0.2,seed=SEED),\n#         #keras.layers.experimental.preprocessing.RandomCrop(HEIGHT//2, WIDTH//2,seed=SEED),\n#         keras.layers.experimental.preprocessing.RandomContrast(0.1,seed=SEED),\n#         keras.layers.experimental.preprocessing.RandomFlip(),\n#         keras.layers.experimental.preprocessing.RandomRotation(0.2,seed=SEED),\n#         #keras.layers.experimental.preprocessing.Resizing(HEIGHT, WIDTH),\n#     ])\n# daug = data_augmentation()\n\n# use RandAugment containing in keras_efficientnet_v2(origin source is https://github.com/tensorflow/models/blob/master/official/vision/ops/augment.py)\nclass RandomProcessImage:\n    def __init__(self, target_shape=(300, 300), magnitude=0, keep_shape=False):\n        self.target_shape, self.magnitude, self.keep_shape = target_shape, magnitude, keep_shape\n        self.target_shape = target_shape if len(target_shape) == 2 else target_shape[:2]\n        if magnitude > 0:\n            from keras_efficientnet_v2 import augment\n\n            translate_const, cutout_const = 100, 40\n#             translate_const = int(target_shape[0] * 10 / magnitude)\n            cutout_const = int(target_shape[0] * 40 / 224)\n            print(\">>>> RandAugment: magnitude = %d, translate_const = %d, cutout_const = %d\" % (magnitude, translate_const, cutout_const))\n            aa = augment.RandAugment(num_layers = 2, magnitude=magnitude, translate_const=translate_const, cutout_const=cutout_const)\n            # aa.available_ops = [\"AutoContrast\", \"Equalize\", \"Invert\", \"Rotate\", \"Posterize\", \"Solarize\", \"Color\", \"Contrast\", \"Brightness\", \"Sharpness\", \"ShearX\", \"ShearY\", \"TranslateX\", \"TranslateY\", \"Cutout\", \"SolarizeAdd\"]\n            # aa.available_ops = [\"AutoContrast\", \"Equalize\", \"Invert\", \"Rotate\", \"Posterize\", \"Solarize\", \"Color\", \"Contrast\", \"Brightness\", \"Sharpness\", \"TranslateX\", \"TranslateY\", \"SolarizeAdd\"]\n            self.process = lambda img: aa.distort(img)\n        elif magnitude == 0:\n            self.process = lambda img: tf.image.random_flip_left_right(img)\n        else:\n            self.process = lambda img: img\n\n    def __call__(self, datapoint):\n        image = datapoint\n        if self.keep_shape:\n            cropped_shape = tf.reduce_min(tf.keras.backend.shape(image)[:2])\n            image = tf.image.random_crop(image, (cropped_shape, cropped_shape, 3))\n\n        input_image = tf.image.resize(image, self.target_shape)\n        input_image = self.process(input_image)\n        # input_image = (tf.cast(input_image, tf.float32) - 127.5) / 128\n        return input_image\n\ndaug = RandomProcessImage((WIDTH, HEIGHT), MAGNITUDE, keep_shape=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:55:32.373584Z","iopub.execute_input":"2022-04-12T12:55:32.373827Z","iopub.status.idle":"2022-04-12T12:55:32.387544Z","shell.execute_reply.started":"2022-04-12T12:55:32.373789Z","shell.execute_reply":"2022-04-12T12:55:32.38672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_train_image(data):\n    img = data['img']\n    cultivar = data['cultivar']\n    name = data['name']\n    target = data['target']\n    # Resize image\n    img = tf.image.resize(img,(WIDTH, HEIGHT),)\n    return img,target\n\ndef load_test_image(data):\n    img = data['img']\n    name = data['name']\n    # Resize image\n    img = tf.image.resize(img,(WIDTH, HEIGHT),)\n#     img = tf.keras.applications.efficientnet.preprocess_input(img)\n    return img,name\n\ndef prepare_train_dataset(train_ds):\n    steps = len(train_ds)//BATCH_SIZE\n    train_ds = train_ds.repeat().shuffle(5000).map(load_train_image,num_parallel_calls=AUTO)\n    train_ds = train_ds.map(lambda x,y:(daug(x),y),num_parallel_calls=AUTO).batch(BATCH_SIZE).prefetch(100)\n    return steps,train_ds\n\ndef prepare_submission_dataset(ds):\n    ds = ds.map(load_test_image,num_parallel_calls=AUTO)\n    ds = ds.batch(BATCH_SIZE*2).prefetch(100)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:55:32.38866Z","iopub.execute_input":"2022-04-12T12:55:32.388886Z","iopub.status.idle":"2022-04-12T12:55:32.402204Z","shell.execute_reply.started":"2022-04-12T12:55:32.38886Z","shell.execute_reply":"2022-04-12T12:55:32.401384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show augmented images","metadata":{}},{"cell_type":"code","source":"show_dataset(train_ds.map(lambda data:(data['img'],data['cultivar'])).map(lambda x,y:(daug(x),y)).batch(9))","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:55:32.403242Z","iopub.execute_input":"2022-04-12T12:55:32.403869Z","iopub.status.idle":"2022-04-12T12:55:36.890263Z","shell.execute_reply.started":"2022-04-12T12:55:32.403834Z","shell.execute_reply":"2022-04-12T12:55:36.889596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model ","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    #Exclude model top layers by set num_classes=0\n    basemodel = keras_efficientnet_v2.EfficientNetV2M(input_shape=(WIDTH, HEIGHT, 3), drop_connect_rate=0.2, num_classes=0, include_preprocessing=True, pretrained=\"imagenet\")\n    basemodel.trainable = True\n\n    image_input = tf.keras.layers.Input(shape=(WIDTH,HEIGHT,3))\n    out = basemodel(image_input)\n    out = tf.keras.layers.GlobalAveragePooling2D()(out)\n    out = tf.keras.layers.Dropout(0.2)(out)\n    out = tf.keras.layers.Dense(100, activation=\"softmax\")(out)\n\n    model = tf.keras.Model(image_input, out)\n\n    model.compile(optimizer=tf.keras.optimizers.Adam(), \n                  loss='sparse_categorical_crossentropy',\n                  metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:55:36.891257Z","iopub.execute_input":"2022-04-12T12:55:36.89173Z","iopub.status.idle":"2022-04-12T12:56:11.385161Z","shell.execute_reply.started":"2022-04-12T12:55:36.891688Z","shell.execute_reply":"2022-04-12T12:56:11.384201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    if not USE_SAVED_MODEL:\n        steps_per_epoch,ds =  prepare_train_dataset(train_ds)\n        callback = tf.keras.callbacks.EarlyStopping(monitor='acc',mode='max', patience=20)\n        ckp_callback = tf.keras.callbacks.ModelCheckpoint(\n                                                    filepath=f'model.h5',\n                                                    save_weights_only=True,\n                                                    monitor='acc',\n                                                    mode='max',\n                                                    options=tf.train.CheckpointOptions(experimental_io_device='/job:localhost'),\n                                                    save_best_only=True)\n        reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='acc',mode='max',factor=0.2,patience=3, min_lr=1e-5)\n        callbacks=[callback,ckp_callback,reduce_lr]\n        # Compile the model\n        model.compile(optimizer=tf.keras.optimizers.Adam(1e-3),\n                      loss=tf.keras.losses.sparse_categorical_crossentropy,\n                      metrics=['acc',])\n\n        history = model.fit(ds,\n                            steps_per_epoch=steps_per_epoch,\n                            epochs=30,\n                            callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T12:56:11.386569Z","iopub.execute_input":"2022-04-12T12:56:11.386827Z","iopub.status.idle":"2022-04-12T13:54:35.97802Z","shell.execute_reply.started":"2022-04-12T12:56:11.386797Z","shell.execute_reply":"2022-04-12T13:54:35.975777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"if not USE_SAVED_MODEL:\n    model.load_weights('model.h5')\nelse:\n    model.load_weights('../input/sorghum100efficientnetbaselinemodel/model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-04-12T13:54:35.983804Z","iopub.execute_input":"2022-04-12T13:54:35.984223Z","iopub.status.idle":"2022-04-12T13:54:44.551633Z","shell.execute_reply.started":"2022-04-12T13:54:35.984147Z","shell.execute_reply":"2022-04-12T13:54:44.550625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(x):\n    return model(x,training=False)\n@tf.function\ndef dist_predict(dist_inputs):\n    res = strategy.run(predict, args=(dist_inputs,))\n    return res","metadata":{"execution":{"iopub.status.busy":"2022-04-12T13:54:44.553612Z","iopub.execute_input":"2022-04-12T13:54:44.553874Z","iopub.status.idle":"2022-04-12T13:54:44.560409Z","shell.execute_reply.started":"2022-04-12T13:54:44.553846Z","shell.execute_reply":"2022-04-12T13:54:44.559799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith strategy.scope():\n    ds = prepare_submission_dataset(test_ds)\n    dist_ds = strategy.experimental_distribute_dataset(ds)\n    \n    all_names = []\n    all_targets = []\n    for img,names in tqdm(dist_ds):\n        preds = dist_predict(img)\n        if tpu:\n            preds = tf.concat(preds.values,axis=0)\n            names = tf.concat(names.values,axis=0)\n        preds = preds.numpy()\n        names = names.numpy()\n        preds = np.argmax(preds,axis=-1)\n        all_targets.extend(list(preds))\n        all_names.extend([s.decode('ascii') for s in names])","metadata":{"execution":{"iopub.status.busy":"2022-04-12T13:54:44.561574Z","iopub.execute_input":"2022-04-12T13:54:44.562274Z","iopub.status.idle":"2022-04-12T13:55:50.307931Z","shell.execute_reply.started":"2022-04-12T13:54:44.56224Z","shell.execute_reply":"2022-04-12T13:55:50.306911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.DataFrame({\n    \"filename\":all_names,\n    \"cultivar\":le.inverse_transform(all_targets)\n})\nsub_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T13:55:50.30939Z","iopub.execute_input":"2022-04-12T13:55:50.309734Z","iopub.status.idle":"2022-04-12T13:55:50.357632Z","shell.execute_reply.started":"2022-04-12T13:55:50.3097Z","shell.execute_reply":"2022-04-12T13:55:50.35664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-12T13:55:50.359105Z","iopub.execute_input":"2022-04-12T13:55:50.359855Z","iopub.status.idle":"2022-04-12T13:55:50.433078Z","shell.execute_reply.started":"2022-04-12T13:55:50.359821Z","shell.execute_reply":"2022-04-12T13:55:50.431861Z"},"trusted":true},"execution_count":null,"outputs":[]}]}