{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA <a id=\"1\"></a>","metadata":{"id":"wRNLR-MQ2i90"}},{"cell_type":"markdown","source":"## Preparing the ground <a id=\"1.1\"></a>","metadata":{"id":"ufcI1cah2o_T"}},{"cell_type":"markdown","source":"### Install and import necessary libraries","metadata":{"id":"1XvcaAzr2rjY"}},{"cell_type":"code","source":"import os\nimport gc\nimport re\n\nimport cv2\nimport math\nimport numpy as np\nimport scipy as sp\nimport pandas as pd\n\nimport tensorflow as tf\nfrom IPython.display import SVG\n# import efficientnet.tfkeras as efn\nfrom keras.utils import plot_model\nimport tensorflow.keras.layers as L\nfrom keras.utils import model_to_dot\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.models import Model\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.applications import DenseNet121\n\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.cm as cm\nfrom sklearn import metrics\nfrom sklearn.preprocessing import MultiLabelBinarizer\nimport matplotlib.pyplot as plt\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\n\ntqdm.pandas()\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\nnp.random.seed(0)\ntf.random.set_seed(0)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"y4ElXcLopnKO","outputId":"72756d56-48f2-46c7-d8d0-56ec9fc395ce","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Setup TPU Config","metadata":{"id":"zOfbl73V6t3p"}},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nGCS_DS_PATH = 'gs://kds-044025978685b91e16a595d7ad1cbace6eb4029e76d4f555679a21cd'","metadata":{"id":"2ZC6VPQHpnMR","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load labels and paths","metadata":{"id":"SuAHc2hu6-Nu"}},{"cell_type":"code","source":"def decode_image(filename, label=None, image_size=(512, 512)):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, image_size)\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.models.load_model(\"../input/plant2021/model_name.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def format_test(st):\n    return GCS_DS_PATH + '/test_images/' + st \n\ntest_dir = '/kaggle/input/plant-pathology-2021-fgvc8/test_images/'\ntest_data = pd.DataFrame(os.listdir(test_dir), columns=['image'])\ntest_paths = test_data.image.apply(format_test).values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(test_paths)\n    .map(decode_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n# X_train, X_test, y_train, y_test = train_test_split(test_paths, test_paths, test_size=0.999, random_state=42)\n# test_dataset = (\n#     tf.data.Dataset\n#     .from_tensor_slices(X_train)\n#     .map(decode_image, num_parallel_calls=AUTO)\n#     .batch(BATCH_SIZE)\n# )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model predicts the leaf diseases with great accuracy. The level of performance is similar to that of DenseNet, as the green bars are very common. The red and blue bars are more prominent in the last (fourth) leaf labeled \"multiple diseases\". This is probably because leaves with multiple diseases may show symptoms of rust and scab as well, thus slightly confusing the model.","metadata":{}},{"cell_type":"code","source":"IMAGE_PATH = \"../input/plant-pathology-2021-fgvc8/train_images/\"\nTRAIN_PATH = \"../input/plant-pathology-2021-fgvc8/train.csv\"\ntrain_data = pd.read_csv(TRAIN_PATH)\naaa = train_data['labels'].value_counts().keys()\ndf = pd.DataFrame()\ndf['label'] = aaa\ndf['labels'] = aaa\ndf['labels'] =  df['labels'].apply(lambda string: string.split(' '))\ns = list(df['labels'])\nmlb = MultiLabelBinarizer()\ntrainx = pd.DataFrame(mlb.fit_transform(s), columns=mlb.classes_, index=df.index)\ntrainx['label'] = aaa\nmerge = {}\nfor i in aaa:\n    merge[tuple(np.array(trainx[trainx['label']==i])[0][0:-1])] = i\nmerge","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generate submission","metadata":{}},{"cell_type":"code","source":"probs_efn = model.predict(test_dataset, verbose=1)\nprint(probs_efn)\nsub = pd.DataFrame(columns = ['image', 'labels'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"arr =  ['complex','frog_eye_leaf_spot',\t'healthy'\t,'powdery_mildew'\t,'rust',\t'scab']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = []\nfor aa in probs_efn:\n    cnt = 0\n    thres = 0.2\n    ok = False\n    while (thres<0.5):\n        m = [0,0,0,0,0,0]\n        ind = np.argwhere(aa > thres)\n        for i in ind:\n            m[i[0]] = 1\n        if tuple(m) in merge:\n            output.append(merge[tuple(m)])\n            ok = True\n            break\n        thres += 0.05\n    if ok == False:\n        output.append('scab')\noutput","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['image'] = test_data['image']\nsub['labels'] = output\nsub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('./submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}