{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Summary\n\nNote book này được sao chép và chỉnh sửa lại từ project của [Nick Kuzmenkov](https://www.kaggle.com/nickuzmenkov)\n* Preprocessing: loại bỏ 77 ảnh bị bị trùng lặp, chuyễn nhãn thành dạng one hot vector, tạo augmentation cho tập dữ liệu và chia tập dữ liệu thành 5 fold.\n* Backbone: EfficientNetB4, B6, B7, `noisy-student` weight.\n* Optimizer: Adam, learning rate of 1e-3, ReduceLROnPlateau\n* Image size: 600x600\n* Augmentations:`albumentations` library.\n\nSố điểm cao nhất đạt được: 0.837\n\n### Các notebook khác của nhóm:\n1. [Revealing Duplicates notebook](https://www.kaggle.com/nvlinhh/int3414-22-n11-revealing-duplicate)\n2. [Preprocessing notebook](https://www.kaggle.com/congnguyen8201/int3414-22-n11-preprocessing)\n3. [Submission notebook](https://www.kaggle.com/congnguyen8201/int3414-22-n11-submission)\n\n### Imports","metadata":{"papermill":{"duration":0.016706,"end_time":"2021-03-25T15:08:44.28578","exception":false,"start_time":"2021-03-25T15:08:44.269074","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/input/efficientnet-keras-dataset/efficientnet_kaggle')","metadata":{"papermill":{"duration":0.023071,"end_time":"2021-03-25T15:08:44.351158","exception":false,"start_time":"2021-03-25T15:08:44.328087","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import KFold\nimport efficientnet.tfkeras as efn\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport os","metadata":{"papermill":{"duration":9.346794,"end_time":"2021-03-25T15:08:53.712259","exception":false,"start_time":"2021-03-25T15:08:44.365465","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hardware configuration","metadata":{"papermill":{"duration":0.014189,"end_time":"2021-03-25T15:08:53.740753","exception":false,"start_time":"2021-03-25T15:08:53.726564","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print('Using tensorflow %s' % tf.__version__)\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    print('Running on TPUv3-8')\nexcept:\n    tpu = None\n    tf.keras.mixed_precision.set_global_policy('mixed_float16')\n    strategy = tf.distribute.get_strategy()\n    print('Running on GPU with mixed precision')\n\nbatch_size = 16 * strategy.num_replicas_in_sync\n\nprint('Number of replicas:', strategy.num_replicas_in_sync)\nprint('Batch size: %.i' % batch_size)","metadata":{"papermill":{"duration":5.782414,"end_time":"2021-03-25T15:08:59.537638","exception":false,"start_time":"2021-03-25T15:08:53.755224","status":"completed"},"tags":[],"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Configurations\nTweak `used folds`, `patience`, `epochs` and other hyperparameters","metadata":{"papermill":{"duration":0.014262,"end_time":"2021-03-25T15:08:59.567709","exception":false,"start_time":"2021-03-25T15:08:59.553447","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG():\n    \n    '''\n    keep these\n    '''\n    strategy = strategy\n    batch_size = batch_size\n    \n    img_size = 600\n    classes = [\n        'complex', \n        'frog_eye_leaf_spot', \n        'powdery_mildew', \n        'rust', \n        'scab']\n    \n    gcs_path_raw = KaggleDatasets().get_gcs_path('pp2021-kfold-tfrecords-0')\n    \n    gcs_path_aug = [\n        KaggleDatasets().get_gcs_path('pp2021-kfold-tfrecords'),\n        KaggleDatasets().get_gcs_path('pp2021-kfold-tfrecords-1'),\n        ]\n    \n    '''\n    tweak these\n    '''\n    seed = 42 # random seed we use for each operation\n    epochs = 100 # maximum number of epochs <-- keep this large as we use EarlyStopping\n    patience = [5, 2] # patience[0] is for EarlyStopping, patience[1] is for ReduceLROnPlateau\n    factor = .1 # new_lr =  lr * factor if patience_count > patience[1]\n    min_lr = 1e-8 # minimum optimizer lr\n    \n    verbose = 2 # set this to 1 to see live progress bar or to 2 when commiting\n    \n    folds = 5 # number of KFold folds\n    used_folds = [0, 1, 2, 3, 4] # number of used folds <-- here we use only the first one","metadata":{"papermill":{"duration":1.950559,"end_time":"2021-03-25T15:09:01.533308","exception":false,"start_time":"2021-03-25T15:08:59.582749","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Helper functions","metadata":{"papermill":{"duration":0.014744,"end_time":"2021-03-25T15:09:01.565357","exception":false,"start_time":"2021-03-25T15:09:01.550613","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def count_data_items(filenames):#dếm lượng phần tử trong dataset\n    return np.sum([int(x[:-6].split('-')[-1]) for x in filenames])\n\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)#chuyển ảnh sang dạng jpeg với đầu ra là rgb\n    image = tf.reshape(image, [CFG.img_size, CFG.img_size, 3])#chuyển dạng ảnh sang 600x600x3\n    image = tf.cast(image, tf.float32) / 255.#The whole math for neural networks is continuous, not discrete, and this is best approximated with floating point numbers.\n     #https://stackoverflow.com/questions/59986353/why-do-i-have-to-convert-uint8-into-float32\n    return image\n\n\nfeature_map = {\n    'image': tf.io.FixedLenFeature([], tf.string),\n    'image_name': tf.io.FixedLenFeature([], tf.string),\n    'complex': tf.io.FixedLenFeature([], tf.int64),\n    'frog_eye_leaf_spot': tf.io.FixedLenFeature([], tf.int64),\n    'powdery_mildew': tf.io.FixedLenFeature([], tf.int64),\n    'rust': tf.io.FixedLenFeature([], tf.int64),\n    'scab': tf.io.FixedLenFeature([], tf.int64),\n    'healthy': tf.io.FixedLenFeature([], tf.int64)}\n\n\ndef read_tfrecord(example, labeled=True):\n    example = tf.io.parse_single_example(example, feature_map)\n#     print(example, '\\n') #ánh xạ keys feature_map sang type tensor\n    image = decode_image(example['image'])\n#     print(image,'\\n')\n\n#nếu ảnh được gán label thì caset label sang dạng tf.float32 onehot vector) nếu chưa thì gán label thành image_name.\n    if labeled:\n        label = [tf.cast(example[x], tf.float32) for x in CFG.classes]\n    else:\n        label = example['image_name']\n    return image, label\n\n\ndef get_dataset(filenames, labeled=True, ordered=True, shuffled=False, \n                repeated=False, cached=False, distributed=True):\n    auto = tf.data.experimental.AUTOTUNE\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=auto)\n    #     dataset = tf.data.TFRecordDataset(filenames)\n    if not ordered:\n        ignore_order = tf.data.Options()\n        ignore_order.experimental_deterministic = False\n        dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(\n        lambda x: read_tfrecord(x, labeled=labeled),\n#         lambda x: read_tfrecord(x, labeled=labeled))\n        num_parallel_calls=auto)\n    if shuffled:\n        dataset = dataset.shuffle(2048, seed=CFG.seed)#xáo trộn tập dataset\n    if repeated:\n        dataset = dataset.repeat()#lặp lại dữ liệu\n    dataset = dataset.batch(CFG.batch_size)#chia dataset thàn các batch\n    if cached:\n        dataset = dataset.cache()#tạo bộ nhớ cached cho dataset\n    dataset = dataset.prefetch(auto)\n    if distributed:\n        dataset = CFG.strategy.experimental_distribute_dataset(dataset)#tạo dạng dataset phù hợp với việc có thể phân phối qua nhiều TPU,GPU khác nhau\n    return dataset\n\n\ndef get_model():\n    model = tf.keras.models.Sequential(name='EfficientNetB7')\n    \n    model.add(efn.EfficientNetB7(\n        include_top=False,\n        input_shape=(CFG.img_size, CFG.img_size, 3),\n        weights='noisy-student',\n        pooling='avg'))\n    \n    model.add(tf.keras.layers.Dense(len(CFG.classes), \n        kernel_initializer=tf.keras.initializers.RandomUniform(seed=CFG.seed),\n        bias_initializer=tf.keras.initializers.Zeros(), name='dense_top'))\n    model.add(tf.keras.layers.Activation('sigmoid', dtype='float32'))\n    \n    return model","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.036101,"end_time":"2021-03-25T15:09:01.616541","exception":false,"start_time":"2021-03-25T15:09:01.58044","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inspect augmented images","metadata":{"papermill":{"duration":0.014766,"end_time":"2021-03-25T15:09:01.69031","exception":false,"start_time":"2021-03-25T15:09:01.675544","status":"completed"},"tags":[]}},{"cell_type":"code","source":"filenames = tf.io.gfile.glob(os.path.join(CFG.gcs_path_aug[0], 'fold_0/*.tfrec'))[:1]\n\ndataset = get_dataset(filenames, ordered=False, distributed=False)\n\nplt.figure(figsize=[15, 15])\n\nfor i, sample in enumerate(dataset.unbatch().take(25).as_numpy_iterator()):\n    plt.subplot(5, 5, i + 1)\n    plt.imshow(sample[0])\n    plt.axis('off')\n    \nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":4.630417,"end_time":"2021-03-25T15:09:06.337","exception":false,"start_time":"2021-03-25T15:09:01.706583","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inspect model","metadata":{"papermill":{"duration":0.05357,"end_time":"2021-03-25T15:09:06.446812","exception":false,"start_time":"2021-03-25T15:09:06.393242","status":"completed"},"tags":[]}},{"cell_type":"code","source":"model = get_model()\nmodel.summary()","metadata":{"_kg_hide-input":true,"papermill":{"duration":9.304484,"end_time":"2021-03-25T15:09:15.806485","exception":false,"start_time":"2021-03-25T15:09:06.502001","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train loop (5 folds CV)","metadata":{"papermill":{"duration":0.053191,"end_time":"2021-03-25T15:09:15.911454","exception":false,"start_time":"2021-03-25T15:09:15.858263","status":"completed"},"tags":[]}},{"cell_type":"code","source":"histories = []\nscores = []\nimage_names = np.empty((0,))\npredicts = np.empty((0, len(CFG.classes)))\n\ncallbacks = [\n    tf.keras.callbacks.EarlyStopping(\n        monitor='val_f1_score', mode='max', \n        patience=CFG.patience[0], restore_best_weights=True),\n    tf.keras.callbacks.ReduceLROnPlateau(\n        monitor='val_f1_score', mode='max',\n        patience=CFG.patience[1], min_lr=CFG.min_lr, verbose=2)]\n\nkfold = KFold(n_splits=CFG.folds, shuffle=True, random_state=CFG.seed)\nfolds = ['fold_0', 'fold_1', 'fold_2', 'fold_3', 'fold_4']\n\n'''\nrun training loop\n'''\nfor i, (train_index, val_index) in enumerate(kfold.split(folds)):\n    \n    '''\n    run only selected folds\n    '''\n    if i in CFG.used_folds:\n        \n        print('=' * 74)\n        print(f'Fold {i}') \n        print('=' * 74)\n        \n        '''\n        reinitialize the system\n        '''\n        if tpu is not None: \n            tf.tpu.experimental.initialize_tpu_system(tpu)\n        \n        '''\n        model setup\n        '''\n        with CFG.strategy.scope():\n            model = get_model()\n            \n            model.compile(\n                loss=tf.keras.losses.BinaryCrossentropy(),\n                optimizer='adam',\n                metrics=[\n                    tf.keras.metrics.BinaryAccuracy(name='acc'), \n                    tfa.metrics.F1Score(\n                        num_classes=len(CFG.classes), \n                        average='macro')])\n            \n        '''\n        data setup\n        '''\n        train_filenames = []\n        for j in train_index:\n            train_filenames += tf.io.gfile.glob(os.path.join(CFG.gcs_path_aug[0], folds[j], '*.tfrec'))\n            train_filenames += tf.io.gfile.glob(os.path.join(CFG.gcs_path_aug[1], folds[j], '*.tfrec'))\n        np.random.shuffle(train_filenames)\n            \n        val_filenames = []\n        for j in val_index:\n            val_filenames += tf.io.gfile.glob(os.path.join(CFG.gcs_path_raw, folds[j], '*.tfrec'))\n\n        train_dataset = get_dataset(\n            train_filenames, \n            ordered=False, shuffled=True, repeated=True)\n        \n        val_dataset = get_dataset(\n            val_filenames, \n            cached=True)\n\n        steps_per_epoch = count_data_items(train_filenames) // (20 * CFG.batch_size)\n        validation_steps = count_data_items(val_filenames) // CFG.batch_size\n        \n        '''\n        fit\n        '''\n        history = model.fit(\n            train_dataset,\n            steps_per_epoch=steps_per_epoch,\n            validation_data=val_dataset,\n            validation_steps=validation_steps,\n            callbacks=callbacks,\n            epochs=CFG.epochs,\n            verbose=CFG.verbose).history\n        \n        '''\n        write out-of-fold predictions\n        '''\n        size = count_data_items(val_filenames)\n        steps = size // CFG.batch_size + 1\n        \n        val_dataset = get_dataset(val_filenames, labeled=False, distributed=False)\n        val_predicts = model.predict(\n            val_dataset.map(lambda x, y: x), \n            steps=steps, \n            verbose=CFG.verbose)[:size]\n        val_image_names = [x.decode() for x in val_dataset.map(lambda x, y: y).unbatch().take(size).as_numpy_iterator()]\n        \n        image_names = np.concatenate((image_names, val_image_names))\n        predicts = np.concatenate((predicts, val_predicts))\n        \n        '''\n        finalize\n        '''\n        model.save_weights(f'model_{i}.h5')\n        histories.append(pd.DataFrame(history))\n        scores.append(histories[-1]['val_f1_score'].max())\n        \n    else:\n        pass","metadata":{"_kg_hide-output":true,"papermill":{"duration":1146.772608,"end_time":"2021-03-25T15:28:22.736556","exception":false,"start_time":"2021-03-25T15:09:15.963948","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Results\nDisplay out-of-fold scores.","metadata":{"papermill":{"duration":0.067111,"end_time":"2021-03-25T15:28:26.459114","exception":false,"start_time":"2021-03-25T15:28:26.392003","status":"completed"},"tags":[]}},{"cell_type":"code","source":"scores_df = pd.DataFrame({\n    'fold': np.arange(len(scores)),\n    'f1': np.round(scores, 4)})\n\nwith pd.option_context('display.max_rows', None):\n    display(scores_df)\n\nprint('CV %.4f' % scores_df['f1'].mean())","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.107115,"end_time":"2021-03-25T15:28:26.636279","exception":false,"start_time":"2021-03-25T15:28:26.529164","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure, axes = plt.subplots(1, 5, figsize=[20, 5])\n\nfor i in range(CFG.folds):\n    \n    try:\n        axes[i].plot(histories[i].loc[:, 'f1_score'], label='train')\n        axes[i].plot(histories[i].loc[:, 'val_f1_score'], label='val')\n        axes[i].legend()\n    except IndexError:\n        pass\n    \n    axes[i].set_title(f'fold {i}')\n    axes[i].set_xlabel('epochs')\n    \nplt.show()","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.725418,"end_time":"2021-03-25T15:28:27.433584","exception":false,"start_time":"2021-03-25T15:28:26.708166","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Write out-of-fold predictions to `oof_predicts.csv` (we will need them later).","metadata":{}},{"cell_type":"code","source":"predicts_df = pd.DataFrame(\n    columns=CFG.classes, \n    data=predicts, \n    index=pd.Index(data=image_names, name='image'))\n\npredicts_df.to_csv('oof_predicts.csv')\ndisplay(predicts_df.head())","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}