{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q ../input/kerasapplications\n!pip install -q '../input/efficientnet-keras-source-code'\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow.keras\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.layers import Convolution2D, MaxPooling2D, Flatten,Dense,Dropout,BatchNormalization, Input\nfrom tensorflow.keras.applications import VGG16, InceptionResNetV2, ResNet50, Xception\nfrom kaggle_datasets import KaggleDatasets\nimport cv2\nfrom PIL import Image\nfrom IPython.display import FileLink\nfrom glob import glob\nimport random\nimport math\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-22T03:43:35.447226Z","iopub.execute_input":"2021-05-22T03:43:35.447582Z","iopub.status.idle":"2021-05-22T03:44:36.381798Z","shell.execute_reply.started":"2021-05-22T03:43:35.447507Z","shell.execute_reply":"2021-05-22T03:44:36.380836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# try:\n#     tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n#     print('Device:', tpu.master())\n#     tf.config.experimental_connect_to_cluster(tpu)\n#     tf.tpu.experimental.initialize_tpu_system(tpu)\n#     strategy = tf.distribute.experimental.TPUStrategy(tpu)\n# except:\n#     strategy = tf.distribute.get_strategy()\n# print('Number of replicas:', strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:59:42.986283Z","iopub.execute_input":"2021-05-26T17:59:42.986764Z","iopub.status.idle":"2021-05-26T17:59:42.991264Z","shell.execute_reply.started":"2021-05-26T17:59:42.986656Z","shell.execute_reply":"2021-05-26T17:59:42.990487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GCS_PATH = KaggleDatasets().get_gcs_path()\n\npath = '../input/plant-pathology-2021-fgvc8/'\ntrain_dir = path + 'train_images/'\ntest_dir = path + 'test_images/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/plant-pathology-2021-fgvc8/train.csv')\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['labels'] = df['labels'].map(lambda values: values.split())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['complete_path'] = train_dir + df['image']\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_names = {0: 'complex', 1: 'powdery_mildew', 2: 'frog_eye_leaf_spot', 3: 'rust', 4: 'scab', 5: 'healthy'}\n\nreverse_train_labels = dict((v,k) for k,v in label_names.items())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list1 = []\nfor i in range(len(df['labels'].values)):\n    for j in range(len(df['labels'].values[i])):\n        df['labels'].values[i][j] = df['labels'].values[i][j].replace(df['labels'].values[i][j], str(reverse_train_labels[df['labels'].values[i][j]]))\nfor i in range(len(df['labels'].values)):\n    list1.append(df['labels'].values[i])\n# print(list1)\ndf['labels'].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['labels'] = df['labels'].map(lambda values: ' '.join(values).strip())\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, val = train_test_split(df, test_size=0.2, random_state=999, shuffle=True)\ntrain.shape, val.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_clean_data(df):\n    targets = []\n    paths = []\n    for _, row in df.iterrows():\n        target_np = np.zeros((6))\n        t = [int(t) for t in row.labels.split()]\n        target_np[t] = 1\n        targets.append(target_np)\n        paths.append(row.complete_path)\n    return np.array(paths), np.array(targets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path, train_target = get_clean_data(train)\nval_path, val_target = get_clean_data(val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Train path shape: {train_path.shape}')\nprint(f'Train target shape: {train_target.shape}')\nprint(f'Val path shape: {val_path.shape}')\nprint(f'Val target shape: {val_target.shape}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = tf.data.Dataset.from_tensor_slices((train_path, train_target))\nval_data = tf.data.Dataset.from_tensor_slices((val_path, val_target))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_size_dim = 400","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_data_train(image_path, label):\n    # load the raw data from the file as a string\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.random_brightness(img, 0.3)\n    img = tf.image.random_flip_left_right(img, seed=None)\n    img = tf.image.random_flip_up_down(img)\n    img = tf.image.resize(img, size=[target_size_dim, target_size_dim])\n    img = tf.cast(img, tf.float32)\n    return img/255, label\n\ndef process_data_valid(image_path, label):\n    # load the raw data from the file as a string\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.resize(img, [target_size_dim,target_size_dim])\n    img = tf.cast(img, tf.float32)\n    return img/255, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = train_data.map(process_data_train, num_parallel_calls=tf.data.experimental.AUTOTUNE)\nvalid_ds = val_data.map(process_data_valid, num_parallel_calls=tf.data.experimental.AUTOTUNE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def configure_for_performance(ds, batch_size = 128):\n#     ds = ds.cache('/kaggle/dump.tfcache') \n    \n    ds = ds.shuffle(buffer_size=200)\n    ds = ds.batch(batch_size)\n    ds = ds.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n    return ds\n\nbatch_size = 64\n\ntrain_ds_batch = configure_for_performance(train_ds, batch_size)\nvalid_ds_batch = valid_ds.batch(batch_size)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install efficientnet\nimport efficientnet.keras as efn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_model = tf.keras.models.load_model(r\"../input/trained-model-for-plant-disease-competition/28_may_plant_model.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = new_model.fit(train_ds_batch, validation_data = valid_ds_batch, epochs = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plotLearningCurve(history,epochs):\n  epochRange = range(1,epochs+1)\n  plt.figure(figsize = (12,6))\n  plt.plot(epochRange,history.history['accuracy'])\n  plt.plot(epochRange,history.history['val_accuracy'])\n  plt.title('Model Accuracy')\n  plt.xlabel('Epoch')\n  plt.ylabel('Accuracy')\n  plt.legend(['Train','Validation'],loc='upper left')\n  plt.show()\n\n  plt.figure(figsize = (12,6))\n  plt.plot(epochRange,history.history['loss'])\n  plt.plot(epochRange,history.history['val_loss'])\n  plt.title('Model Loss')\n  plt.xlabel('Epoch')\n  plt.ylabel('Loss')\n  plt.legend(['Train','Validation'],loc='upper left')\n  plt.show()\n\n# plotLearningCurve(history,20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = pd.read_csv(path + 'sample_submission.csv')\ntest_set.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_imgs = ['../input/plant-pathology-2021-fgvc8/test_images/{}'.format(x) for x in list(test_set.image)]\nprint(test_imgs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.DataFrame(np.array(test_imgs), columns=['Path'])\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = tf.data.Dataset.from_tensor_slices((df_test.Path.values))\n\ndef process_test(image_path):\n    # load the raw data from the file as a string\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels=3)\n    img = tf.image.random_brightness(img, 0.3)\n    img = tf.image.random_flip_left_right(img, seed=None)\n    img = tf.image.random_flip_up_down(img)\n    img = tf.image.resize(img, size=[target_size_dim, target_size_dim])\n    img = tf.cast(img, tf.float32)\n    return img/255\n    \ntest_ds = test_ds.map(process_test, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(batch_size*2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nfor i in range(6):\n    \n    pred_test = new_model.predict(test_ds, workers=16, verbose=1)\n    preds.append(pred_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y = np.mean(preds, axis=0)\nprint(pred_y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = {0: 0.25, 1: 0.30, 2: 0.23, 3: 0.30, 4: 0.30, 5: 0.30}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_string = []\nfor line in pred_y:\n    s = ''\n    for i in range(6):\n        if line[i] > threshold[i]:\n            s = s + label_names[i] + ' '\n\n    pred_string.append(s)\nprint(pred_string)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['image'] = df_test.Path.str.split('/').str[-1]\ndf_test['labels'] = pred_string\n# df_test['labels'] = df_test['labels'].replace(label_names)\ndf_test= df_test[['image','labels']]\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir(r'/kaggle/working')\n\ndf_test.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FileLink(r'submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}