{"cells":[{"metadata":{},"cell_type":"markdown","source":"https://www.kaggle.com/dimitreoliveira/flower-classification-with-tpus-eda-and-baseline/notebook\n\n이 베이스라인을 따라서 해보자"},{"metadata":{},"cell_type":"markdown","source":"# Dependencies"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math, os, re, warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\n# kaggle_datasets가 뭐였는지 youtube에서 설명해줬는데 기억이 안난다\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.utils import class_weight\nfrom sklearn.metrics import classification_report, confusion_matrix\n\nimport tensorflow as tf\nfrom tensorflow.keras import optimizers, applications, Sequential, layers\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\n\n# 랜덤시드를 특정 값으로 고정시켜주는 부분\ndef seed_everything(seed=0):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISSTIC_OPS'] = '1'\n    \nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TPU, GPU 하드웨어를 찾는다\n# 적절한 distribution strategy를 리턴한다고 하는데\n# tpu인지 gpu인지 설정한다는 소리겠지\n\ntry :\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU', tpu.master())\nexcept ValueError:\n    tpu = None\n\n# TPU가 잡힌다면\nif tpu :\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse :\n    strategy = tf.distribute.get_strategy()\n    \nprint('RELICAS :', strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Model parameter"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16 * strategy.num_replicas_in_sync\nWARMUP_EPOCHS = 3\nWARMUP_LEARNING_RATE = 1e-4 * strategy.num_replicas_in_sync\nEPOCHS = 20\nLEARNING_RATE = 3e-5 * strategy.num_replicas_in_sync\nHEIGHT = 512\nWIDTH = 512\nCHANNELS = 3\nN_CLASSES = 104\nES_PATIENCE = 6\nRLROP_PATIENCE = 3\nDECAY_DROP = 0.3\n\nmodel_path = 'DenseNet201_%sx%s.h5'%(HEIGHT, WIDTH)\n\nGCS_PATH =  KaggleDatasets().get_gcs_path() + '/tfrecords-jpeg-%sx%s'%(HEIGHT, WIDTH)\n\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')\n\n# 이정도 되는 클래스들이면 파일로 전달해줘도 좋았을 텐데\nCLASSES = [\n    'pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', \n    'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', \n    'globe thistle', 'snapdragon', \"colt's foot\", 'king protea', 'spear thistle', \n    'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', \n    'balloon flower', 'giant white arum lily', 'fire lily', 'pincushion flower', \n    'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', \n    'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william', \n    'carnation', 'garden phlox', 'love in the mist', 'cosmos',  'alpine sea holly', \n    'ruby-lipped cattleya', 'cape flower', 'great masterwort',  'siam tulip', \n    'lenten rose', 'barberton daisy', 'daffodil',  'sword lily', 'poinsettia', \n    'bolero deep blue',  'wallflower', 'marigold', 'buttercup', 'daisy', \n    'common dandelion', 'petunia', 'wild pansy', 'primula',  'sunflower', \n    'lilac hibiscus', 'bishop of llandaff', 'gaura',  'geranium', 'orange dahlia', \n    'pink-yellow dahlia', 'cautleya spicata',  'japanese anemone', 'black-eyed susan', \n    'silverbush', 'californian poppy',  'osteospermum', 'spring crocus', 'iris', \n    'windflower',  'tree poppy', 'gazania', 'azalea', 'water lily',  'rose', \n    'thorn apple', 'morning glory', 'passion flower',  'lotus', 'toad lily', \n    'anthurium', 'frangipani',  'clematis', 'hibiscus', 'columbine', 'desert-rose', \n    'tree mallow', 'magnolia', 'cyclamen ', 'watercress',  'canna lily', \n    'hippeastrum ', 'bee balm', 'pink quill',  'foxglove', 'bougainvillea', \n    'camellia', 'mallow',  'mexican petunia',  'bromelia', 'blanket flower', \n    'trumpet creeper',  'blackberry lily', 'common tulip', 'wild rose']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Datasets utility functions\nAUTO = tf.data.experimental.AUTOTUNE # instructs the API to read from multiple files if available.\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef data_augment(image, label):\n    image = tf.image.random_flip_left_right(image, seed=seed)\n    image = tf.image.random_flip_up_down(image, seed=seed)\n    image = tf.image.random_saturation(image, lower=0, upper=2, seed=seed)\n#     image = tf.image.random_contrast(image, lower=.8, upper=2, seed=seed)\n#     image = tf.image.random_brightness(image, max_delta=.2, seed=seed)\n    image = tf.image.random_crop(image, size=[int(HEIGHT*.8), int(WIDTH*.8), CHANNELS], seed=seed)\n\n    return image, label\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_training_dataset_preview(ordered=True):\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualization utility functions\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n    \n# Visualize model predictions\ndef dataset_to_numpy_util(dataset, N):\n    dataset = dataset.unbatch().batch(N)\n    for images, labels in dataset:\n        numpy_images = images.numpy()\n        numpy_labels = labels.numpy()\n        break;  \n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    label = np.argmax(label, axis=-1)\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], str(correct), ', shoud be ' if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower_eval(image, title, subplot, red=False):\n    plt.subplot(subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    plt.title(title, fontsize=14, color='red' if red else 'black')\n    return subplot+1\n\ndef display_9_images_with_predictions(images, predictions, labels):\n    subplot=331\n    plt.figure(figsize=(13,13))\n    for i, image in enumerate(images):\n        title, correct = title_from_label_and_target(predictions[i], labels[i])\n        subplot = display_one_flower_eval(image, title, subplot, not correct)\n        if i >= 8:\n            break;\n              \n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# EDA"},{"metadata":{},"cell_type":"markdown","source":"## About the datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\ntrain_dataset = get_training_dataset_preview(ordered=True)\ny_train = next(iter(train_dataset.unbatch().map(lambda image, label: label).batch(NUM_TRAINING_IMAGES))).numpy()\nprint('Number of training images %d' % NUM_TRAINING_IMAGES)\n\n# Validation data\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nvalid_dataset = get_validation_dataset(ordered=True)\ny_valid = next(iter(valid_dataset.unbatch().map(lambda image, label: label).batch(NUM_VALIDATION_IMAGES))).numpy()\nprint('Number of validation images %d' % NUM_VALIDATION_IMAGES)\n\n# Test data\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint('Number of test images %d' % NUM_TEST_IMAGES)\ntest_dataset = get_test_dataset(ordered=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## First let's look at some samples from each set"},{"metadata":{},"cell_type":"markdown","source":"### Train samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"display_batch_of_images(next(iter(train_dataset.unbatch().batch(20))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Validation samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"display_batch_of_images(next(iter(valid_dataset.unbatch().batch(20))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"display_batch_of_images(next(iter(test_dataset.unbatch().batch(20))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"사진의 퀄리티가 매우 높다.\n이러한 사진을 모델에 입력을 하면 일반화하기 쉽고, 무튼 더 좋다.\nTPU의 장점이라고 볼 수 있겠다."},{"metadata":{},"cell_type":"markdown","source":"### Label distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_agg = np.asarray([[label, (y_train == index).sum()] for index, label in enumerate(CLASSES)])\nvalid_agg = np.asarray([[label, (y_valid == index).sum()] for index, label in enumerate(CLASSES)])\n\nfig, (ax1, ax2) = plt.subplots(2, 1, figsize=(24, 64))\n\nax1 = sns.barplot(x=train_agg[...,1], y=train_agg[...,0], order=CLASSES, ax=ax1)\nax1.set_title('Train', fontsize=30)\nax1.tick_params(labelsize=16)\n\nax2 = sns.barplot(x=valid_agg[...,1], y=valid_agg[...,0], order=CLASSES, ax=ax2)\nax2.set_title('Validation', fontsize=30)\nax2.tick_params(labelsize=16)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"데이터셋의 분포는 많이 불균형스럽지만, train과 validation사이에서는 비슷한 분포를 보이고 있다."},{"metadata":{},"cell_type":"markdown","source":"# Model\n- 사전학습모델 DenseNet201을 사용\n- classification하는 부분은 따로 만들어서 사용 (```include_top=False```)\n- 모델의 가중치는 freeze (```trainable=False```)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(input_shape, N_CLASSES) :\n    base_model = applications.DenseNet201(weights='imagenet',\n                                         include_top=False,\n                                         input_shape=input_shape)\n    \n    base_model.trainable = False  # Freeze layers\n    model = tf.keras.Sequential([\n        base_model,\n        layers.GlobalAveragePooling2D(),\n        layers.Dense(N_CLASSES, activation='softmax')\n    ])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Warmup top layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = create_model((None, None, CHANNELS), N_CLASSES)\n    \nmetric_list = ['sparse_categorical_accuracy']\n\noptimizer = optimizers.Adam(lr=WARMUP_LEARNING_RATE)\nmodel.compile(optimizer=optimizer,\n             loss = 'sparse_categorical_crossentropy',\n             metrics=metric_list)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nwarmup_history = model.fit(x=get_training_dataset(),\n                           steps_per_epoch = STEPS_PER_EPOCH,\n                           validation_data = get_validation_dataset(),\n                           epochs=WARMUP_EPOCHS,\n                           verbose=2).history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"![image](https://user-images.githubusercontent.com/48716219/93314461-b9fc3680-f844-11ea-8d80-b9afde171905.png)\nTPU가 일을 하고 있다. 신기하다.","attachments":{}},{"metadata":{},"cell_type":"markdown","source":"### Learning rate schedule\n- 학습률 조정하는 부분"},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 0.00000001\nLR_MIN = 0.000001\nLR_MAX = LEARNING_RATE\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch) :\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    \n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)\nprint(\"Learning rate shcedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fine-tune all layers\n- Fine tuing을 하는 부분인데 무엇을 건드릴까\n- freeze 했던 layer들을 unfreeze시키고\n- LearnigRateScheduler, EarlyStopping 등 콜백함수 사용"},{"metadata":{"trusted":true},"cell_type":"code","source":"# 모델의 모든 layer들을 객채로 하나씩 반복문으로 돌린다\nfor layer in model.layers:\n    layer.trainable = True # Unfreeze layers\n\ncheckpoint = ModelCheckpoint(model_path, monitor='val_loss', mode='min', save_best_only=True)\nes = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, \n                   restore_best_weights=True, verbose=1)\nlr_callback = LearningRateScheduler(lrfn, verbose=1)\n\ncallback_list = [checkpoint, es, lr_callback]\n\noptimizer = optimizers.Adam(lr=LEARNING_RATE)\nmodel.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=metric_list)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(x=get_training_dataset(), \n                    steps_per_epoch=STEPS_PER_EPOCH, \n                    validation_data=get_validation_dataset(),\n                    callbacks=callback_list,\n                    epochs=EPOCHS, \n                    verbose=2).history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"TPU가 실행하기전 CPU의 사용량이 100퍼센트가 되고, 그러고 잠시후 CPU 사용량이 줄어들면서 TPU를 사용하기 시작  \n그런데 TPU까지 올리기까지는 CPU를 사용해서 올리는 건가.. TPU을 바로 사용하진 않고 사용 전에 대기 시간이 상당히 길다  \n> 큰 사이즈의 데이터를 학습 하고, 이러한 데이터에 대한 학습은 빠르지만(x) 학습을 시작하기 전까지 시간이 많이 걸린다  \n> 지금보니까 학습자체도 빠른거는 아닌 것 같다. 엄청 큰 데이터를 통째로 넣어서 학습할뿐...20에폭을 돌리는데도 시간은 꽤 걸린다  \n> MXU : ?? , TPU 사용량을 퍼센트로 표현하는 것 같은데 10퍼센트를 넘지 않음, 보다 큰 데이터도 가능하다는 소리일 거 같다.  \n\n- 5번째 에폭부터 accuracy는 90이 넘게 출력, 시작은 50\n- 학습을 잘 되는 것 같다. val_loss를 봐도 오버피팅 되는 현상은 없음."},{"metadata":{},"cell_type":"markdown","source":"### Model Loss Graph"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_metrics(history, metric_list):\n    fig, axes = plt.subplots(len(metric_list), 1, sharex='col', figsize=(24, 12))\n    axes = axes.flatten()\n    \n    for index, metric in enumerate(metric_list):\n        axes[index].plot(history[metric], label='Train %s' % metric)\n        axes[index].plot(history['val_%s' % metric], label='Validation %s' % metric)\n        axes[index].legend(loc='best', fontsize=16)\n        axes[index].set_title(metric)\n\n    plt.xlabel('Epochs', fontsize=16)\n    sns.despine()\n    plt.show()\n\nplot_metrics(history, metric_list=['loss', 'sparse_categorical_accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Evaluation"},{"metadata":{},"cell_type":"markdown","source":"## Train set"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = train_dataset.map(lambda image, label : image)\ntrain_preds = model.predict(x_train)\ntrain_preds = np.argmax(train_preds, axis=-1)\n\nprint(classification_report(y_train, train_preds, target_names=CLASSES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_valid = valid_dataset.map(lambda image, label : image)\nvalid_preds = model.predict(x_valid)\nvalid_preds = np.argmax(valid_preds, axis=-1)\n\nprint(classification_report(y_valid, valid_preds, target_names=CLASSES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Confusion Matrix"},{"metadata":{},"cell_type":"markdown","source":"### Train\n- confusion matrix는 3개의 부분으로 나눴다고 한다.\n- 클래스들을 1~34, 35~69, 그리고 나머지의 부분으로"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(24, 45))\n\ntrain_cfn_matrix = confusion_matrix(y_train, train_preds, labels=range(len(CLASSES)))\ntrain_cfn_matrix = (train_cfn_matrix.T / train_cfn_matrix.sum(axis=1)).T\n\ntrain_df_cm1 = pd.DataFrame(train_cfn_matrix[:34], index=CLASSES[:34], columns=CLASSES)\ntrain_df_cm2 = pd.DataFrame(train_cfn_matrix[34:68], index=CLASSES[34:68], columns=CLASSES)\ntrain_df_cm3 = pd.DataFrame(train_cfn_matrix[68:], index=CLASSES[68:], columns=CLASSES)\n\nsns.heatmap(train_df_cm1, cmap=\"Blues\", ax=ax1).set_title('Train (1:35)', fontsize=30)\nsns.heatmap(train_df_cm2, cmap=\"Blues\", ax=ax2).set_title('Train (35:69)', fontsize=30)\nsns.heatmap(train_df_cm3, cmap=\"Blues\", ax=ax3).set_title('Train (69:)', fontsize=30)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(24, 45))\n\nvalid_cfn_matrix = confusion_matrix(y_valid, valid_preds, labels=range(len(CLASSES)))\nvalid_cfn_matrix = (valid_cfn_matrix.T / valid_cfn_matrix.sum(axis=1)).T\n\nvalid_df_cm1 = pd.DataFrame(valid_cfn_matrix[:34], index=CLASSES[:34], columns=CLASSES)\nvalid_df_cm2 = pd.DataFrame(valid_cfn_matrix[34:68], index=CLASSES[34:68], columns=CLASSES)\nvalid_df_cm3 = pd.DataFrame(valid_cfn_matrix[68:], index=CLASSES[68:], columns=CLASSES)\n\nsns.heatmap(valid_df_cm1, cmap=sns.cubehelix_palette(8), ax=ax1).set_title('Validation (1:35)', fontsize=30)\nsns.heatmap(valid_df_cm2, cmap=sns.cubehelix_palette(8), ax=ax2).set_title('Validation (35:69)', fontsize=30)\nsns.heatmap(valid_df_cm3, cmap=sns.cubehelix_palette(8), ax=ax3).set_title('Validation (69:)', fontsize=30)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualize Predictions"},{"metadata":{},"cell_type":"markdown","source":"### Train Set"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_samp, y_train_samp = dataset_to_numpy_util(train_dataset, 9)\ntrain_samp_preds = model.predict(x_train_samp, batch_size=9)\ndisplay_9_images_with_predictions(x_train_samp, train_samp_preds, y_train_samp)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Validation Set"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_valid_samp, y_valid_samp = dataset_to_numpy_util(valid_dataset, 9)\nvalid_samp_preds = model.predict(x_valid_samp, batch_size=9)\ndisplay_9_images_with_predictions(x_valid_samp, valid_samp_preds, y_valid_samp)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Set Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = test_dataset.map(lambda image, idnum: image)\ntest_preds = model.predict(x_test)\ntest_preds = np.argmax(test_preds, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids_ds = test_dataset.map(lambda image, idnum:idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\nsubmission = pd.DataFrame(test_ids, columns=['id'])\nsubmission['label'] = test_preds\nsubmission.to_csv('submission.csv', index=False)\ndisplay(submission.head(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm Den* ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}