{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":30628,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install seaborn --upgrade\n!pip install plotly","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:44:57.115225Z","iopub.execute_input":"2024-01-08T19:44:57.115438Z","iopub.status.idle":"2024-01-08T19:45:15.166847Z","shell.execute_reply.started":"2024-01-08T19:44:57.115415Z","shell.execute_reply":"2024-01-08T19:45:15.166009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport math, re,os\nimport pandas as pd\nfrom matplotlib import cm\nimport numpy as np\nimport random\nimport plotly.express as px\nimport tensorflow as tf\nprint(tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:15.168391Z","iopub.execute_input":"2024-01-08T19:45:15.168707Z","iopub.status.idle":"2024-01-08T19:45:29.888664Z","shell.execute_reply.started":"2024-01-08T19:45:15.168677Z","shell.execute_reply":"2024-01-08T19:45:29.887904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n print(\"Running on TPU : \",tpu.master())\nexcept ValueError:\n tpu = None\nif tpu:\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n strategy = tf.distribute.get_strategy()\nprint(\"REPLICAS: \",strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:29.889611Z","iopub.execute_input":"2024-01-08T19:45:29.890093Z","iopub.status.idle":"2024-01-08T19:45:37.666448Z","shell.execute_reply.started":"2024-01-08T19:45:29.890061Z","shell.execute_reply":"2024-01-08T19:45:37.665482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')\nprint(GCS_DS_PATH)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:37.668622Z","iopub.execute_input":"2024-01-08T19:45:37.668927Z","iopub.status.idle":"2024-01-08T19:45:37.679385Z","shell.execute_reply.started":"2024-01-08T19:45:37.668900Z","shell.execute_reply":"2024-01-08T19:45:37.678585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [512,512]\nGCS_PATH = GCS_DS_PATH + '/tfrecords-jpeg-512x512'","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:37.680280Z","iopub.execute_input":"2024-01-08T19:45:37.680538Z","iopub.status.idle":"2024-01-08T19:45:37.693829Z","shell.execute_reply.started":"2024-01-08T19:45:37.680495Z","shell.execute_reply":"2024-01-08T19:45:37.693132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:37.694748Z","iopub.execute_input":"2024-01-08T19:45:37.694974Z","iopub.status.idle":"2024-01-08T19:45:37.703298Z","shell.execute_reply.started":"2024-01-08T19:45:37.694951Z","shell.execute_reply":"2024-01-08T19:45:37.702553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH+'/test/*.tfrec')\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:37.704212Z","iopub.execute_input":"2024-01-08T19:45:37.704475Z","iopub.status.idle":"2024-01-08T19:45:37.738363Z","shell.execute_reply.started":"2024-01-08T19:45:37.704450Z","shell.execute_reply":"2024-01-08T19:45:37.737594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:37.739403Z","iopub.execute_input":"2024-01-08T19:45:37.739674Z","iopub.status.idle":"2024-01-08T19:45:37.747912Z","shell.execute_reply.started":"2024-01-08T19:45:37.739637Z","shell.execute_reply":"2024-01-08T19:45:37.747134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea', 'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood', 'globe thistle', # 00 - 09\n 'snapdragon', \"colt's foot\", 'king protea', 'spear thistle', 'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower', 'giant white arum lily', # 10 - 19\n 'fire lily', 'pincushion flower', 'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william', # 20 - 29\n 'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip', 'lenten rose', # 30 - 39\n 'barberton daisy', 'daffodil', 'sword lily', 'poinsettia', 'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion', # 40 - 49\n 'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia', # 50 - 59\n 'cautleya spicata', 'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy', # 60 - 69\n 'gazania', 'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily', 'anthurium', # 70 - 79\n 'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily', # 80 - 89\n 'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower', # 90 - 99\n 'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose'] # 100 - 103\n\n\ndef decode_image(image_data):\n image = tf.image.decode_jpeg(image_data, channels=3)\n image = tf.cast(image, tf.float32) / 255.0 # convert image to floats in [0, 1] range\n image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n return image\n\ndef read_labeled_tfrecord(example):\n LABELED_TFREC_FORMAT = {\n \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n \"class\": tf.io.FixedLenFeature([], tf.int64), # shape [] means single element\n }\n example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n image = decode_image(example['image'])\n label = tf.cast(example['class'], tf.int32)\n return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n UNLABELED_TFREC_FORMAT = {\n \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n \"id\": tf.io.FixedLenFeature([], tf.string), # shape [] means single element\n # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n }\n example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n image = decode_image(example['image'])\n idnum = example['id']\n return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n # Read from TFRecords. For optimal performance, reading from multiple files at once and\n # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n ignore_order = tf.data.Options()\n if not ordered:\n ignore_order.experimental_deterministic = False # disable order, increase speed. #See Note 2.3 above 😀\n\n dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files. #See Note 2.2 above 😀\n dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n return dataset","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:37.748960Z","iopub.execute_input":"2024-01-08T19:45:37.749245Z","iopub.status.idle":"2024-01-08T19:45:37.762525Z","shell.execute_reply.started":"2024-01-08T19:45:37.749216Z","shell.execute_reply":"2024-01-08T19:45:37.761742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_augment(image,label):\n image=tf.image.random_flip_left_right(image)\n return image,label\n\ndef get_training_dataset():\n dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n dataset = dataset.map(data_augment,num_parallel_calls=AUTO)\n dataset = dataset.repeat() # the training dataset must repeat for several epochs\n dataset = dataset.shuffle(count_data_items(TRAINING_FILENAMES))\n dataset = dataset.batch(BATCH_SIZE)\n dataset = dataset.prefetch(AUTO)\n return dataset\n\n\ndef get_validation_dataset(ordered=False):\n dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n dataset = dataset.batch(BATCH_SIZE)\n dataset = dataset.cache()\n dataset = dataset.prefetch(AUTO)\n return dataset\n\ndef get_test_dataset(ordered=False):\n dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n dataset = dataset.batch(BATCH_SIZE)\n dataset = dataset.prefetch(AUTO)\n return dataset\n\ndef count_data_items(filenames):\n n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nprint(NUM_TEST_IMAGES,NUM_TRAINING_IMAGES,NUM_VALIDATION_IMAGES)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:37.765180Z","iopub.execute_input":"2024-01-08T19:45:37.765415Z","iopub.status.idle":"2024-01-08T19:45:37.777007Z","shell.execute_reply.started":"2024-01-08T19:45:37.765392Z","shell.execute_reply":"2024-01-08T19:45:37.776261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_train = get_training_dataset()\nds_valid = get_validation_dataset()\nds_test = get_test_dataset()\n\nprint(ds_train)\nprint(ds_test)\nprint(ds_valid)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:37.777845Z","iopub.execute_input":"2024-01-08T19:45:37.778054Z","iopub.status.idle":"2024-01-08T19:45:38.042530Z","shell.execute_reply.started":"2024-01-08T19:45:37.778032Z","shell.execute_reply":"2024-01-08T19:45:38.041539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.set_printoptions(threshold=15, linewidth=80)\nprint(\"Training data Shapes: \")\nfor image, label in ds_train.take(3):\n print(image.numpy().shape, label.numpy().shape)\n print(\"labels\",label.numpy(),len(label.numpy()))\n# print(\"Images\",image.numpy())","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:38.043688Z","iopub.execute_input":"2024-01-08T19:45:38.043957Z","iopub.status.idle":"2024-01-08T19:45:51.139888Z","shell.execute_reply.started":"2024-01-08T19:45:38.043930Z","shell.execute_reply":"2024-01-08T19:45:51.138665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Test data Shapes: \")\nfor image, ids in ds_test.take(3):\n print(image.numpy().shape, ids.numpy().shape)\nprint(\"ids\",ids.numpy())","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:51.141048Z","iopub.execute_input":"2024-01-08T19:45:51.141349Z","iopub.status.idle":"2024-01-08T19:45:52.767839Z","shell.execute_reply.started":"2024-01-08T19:45:51.141321Z","shell.execute_reply":"2024-01-08T19:45:52.766905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\n\ndef batch_to_numpy_images_and_labels(data):\n images, labels = data\n numpy_images = images.numpy()\n numpy_labels = labels.numpy()\n if numpy_labels.dtype == object: # binary string in this case,these are image ID strings\n numpy_labels = [None for _ in enumerate(numpy_images)]\n # If no labels, only image IDs, return None for labels (this is the case for test data)\n return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n if correct_label is None:\n return CLASSES[label], True\n correct = (label == correct_label)\n return \"{} [{}{}{}]\".format(CLASSES[label], \n 'OK' if correct else 'NO', \n u\"\\u2192\" if not correct else '',\n CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n plt.subplot(*subplot)\n plt.axis('off')\n plt.imshow(image)\n if len(title) > 0:\n plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n return (subplot[0], subplot[1], subplot[2]+1)\n \ndef display_batch_of_images(databatch, predictions=None, display_mismatches_only=False):\n \"\"\"This will work with:\n display_batch_of_images(images)\n display_batch_of_images(images, predictions)\n display_batch_of_images((images, labels))\n display_batch_of_images((images, labels), predictions)\n \"\"\"\n # data\n images, labels = batch_to_numpy_images_and_labels(databatch)\n if labels is None:\n labels = [None for _ in enumerate(images)]\n \n # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n rows = int(math.sqrt(len(images)))\n cols = len(images)//rows\n \n # size and spacing\n FIGSIZE = 13.0\n SPACING = 0.1\n subplot=(rows,cols,1)\n if rows < cols:\n plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n else:\n plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n \n # display\n for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n title = '' if label is None else CLASSES[label]\n correct = True\n if predictions is not None:\n title, correct = title_from_label_and_target(predictions[i], label)\n dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n if display_mismatches_only:\n if predictions[i] != label:\n subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n else: \n subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n \n #layout\n plt.tight_layout()\n if label is None and predictions is None:\n plt.subplots_adjust(wspace=0, hspace=0)\n else:\n plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n plt.show()\n\n\ndef display_training_curves(training, validation, title, subplot):\n if subplot%10==1: # set up the subplots on the first call\n plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n plt.tight_layout()\n ax = plt.subplot(subplot)\n ax.set_facecolor('#F8F8F8')\n ax.plot(training)\n ax.plot(validation)\n ax.set_title('model '+ title)\n ax.set_ylabel(title)\n #ax.set_ylim(0.28,1.05)\n ax.set_xlabel('epoch')\n ax.legend(['train', 'valid.'])\n\ndef display_training_curves_v2(training, validation, learning_rate_list, title, subplot):\n if subplot%10==1: # set up the subplots on the first call\n plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n plt.tight_layout()\n ax = plt.subplot(subplot)\n ax.set_facecolor('#F8F8F8')\n ax.plot(training)\n ax.plot(validation)\n ax.set_title('model '+ title)\n ax.set_ylabel(title, color='b')\n #ax.set_ylim(0.28,1.05)\n ax.set_xlabel('epoch')\n ax.legend(['train', 'valid.', 'learning rate']) \n \n ax2 = ax.twinx()\n ax2.plot(learning_rate_list, 'g-')\n ax2.set_ylabel('learning rate', color='g')","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:52.769026Z","iopub.execute_input":"2024-01-08T19:45:52.769325Z","iopub.status.idle":"2024-01-08T19:45:52.788440Z","shell.execute_reply.started":"2024-01-08T19:45:52.769296Z","shell.execute_reply":"2024-01-08T19:45:52.787647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_iter = iter(ds_train.unbatch().batch(20))","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:52.789330Z","iopub.execute_input":"2024-01-08T19:45:52.789562Z","iopub.status.idle":"2024-01-08T19:45:52.835378Z","shell.execute_reply.started":"2024-01-08T19:45:52.789538Z","shell.execute_reply":"2024-01-08T19:45:52.834631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_batch = next(ds_iter)\ndisplay_batch_of_images(one_batch)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:45:52.836238Z","iopub.execute_input":"2024-01-08T19:45:52.836465Z","iopub.status.idle":"2024-01-08T19:46:02.681856Z","shell.execute_reply.started":"2024-01-08T19:45:52.836441Z","shell.execute_reply":"2024-01-08T19:46:02.680926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 3\ncol = 4\nall_elements = get_training_dataset().unbatch()\none_element = tf.data.Dataset.from_tensors(next(iter(all_elements)))\n# Map the images to the data augmentation function for image processing\naugmented_element = one_element.repeat().map(data_augment).batch(row * col)\n\nfor (img, label) in augmented_element:\n plt.figure(figsize = (15, int(15 * row / col)))\n for j in range(row * col):\n plt.subplot(row, col, j + 1)\n plt.axis('off')\n plt.imshow(img[j, ])\n plt.show()\n break","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:46:02.682989Z","iopub.execute_input":"2024-01-08T19:46:02.683287Z","iopub.status.idle":"2024-01-08T19:46:17.186793Z","shell.execute_reply.started":"2024-01-08T19:46:02.683256Z","shell.execute_reply":"2024-01-08T19:46:17.185594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tuning4\nSEED = 2020\n\ndef random_blockout(img, sl=0.1, sh=0.2, rl=0.4):\n p=random.random()\n if p>=0.25:\n w, h, c = IMAGE_SIZE[0], IMAGE_SIZE[1], 3\n origin_area = tf.cast(h*w, tf.float32)\n\n e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)), tf.int32)\n e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)), tf.int32)\n\n e_height_h = tf.minimum(e_size_h, h)\n e_width_h = tf.minimum(e_size_h, w)\n\n erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)\n erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)\n\n erase_area = tf.zeros(shape=[erase_height, erase_width, c])\n erase_area = tf.cast(erase_area, tf.uint8)\n\n pad_h = h - erase_height\n pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n pad_bottom = pad_h - pad_top\n\n pad_w = w - erase_width\n pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n pad_right = pad_w - pad_left\n\n erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n erase_mask = tf.squeeze(erase_mask, axis=0)\n erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))\n\n return tf.cast(erased_img, img.dtype)\n else:\n return tf.cast(img, img.dtype)\n\n \ndef data_augment_v2(image, label):\n # Thanks to the dataset.prefetch(AUTO) statement in the next function (below), this happens essentially for free on TPU. \n # Data pipeline code is executed on the \"CPU\" part of the TPU while the TPU itself is computing gradients.\n \n flag = random.randint(1,3)\n coef_1 = random.randint(70, 90) * 0.01\n coef_2 = random.randint(70, 90) * 0.01\n \n if flag == 1:\n image = tf.image.random_flip_left_right(image, seed=SEED)\n elif flag == 2:\n image = tf.image.random_flip_up_down(image, seed=SEED)\n else:\n image = tf.image.random_crop(image, [int(IMAGE_SIZE[0]*coef_1), int(IMAGE_SIZE[0]*coef_2), 3],seed=SEED)\n \n image = random_blockout(image)\n \n return image, label ","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:46:17.188045Z","iopub.execute_input":"2024-01-08T19:46:17.188350Z","iopub.status.idle":"2024-01-08T19:46:17.200057Z","shell.execute_reply.started":"2024-01-08T19:46:17.188320Z","shell.execute_reply":"2024-01-08T19:46:17.199344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"augmented_element = one_element.repeat().map(data_augment_v2).batch(row * col)\n\nfor (img, label) in augmented_element:\n plt.figure(figsize = (15, int(15 * row / col)))\n for j in range(row * col):\n plt.subplot(row, col, j + 1)\n plt.axis('off')\n plt.imshow(img[j, ])\n plt.show()\n break","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:46:17.200918Z","iopub.execute_input":"2024-01-08T19:46:17.201179Z","iopub.status.idle":"2024-01-08T19:46:19.006560Z","shell.execute_reply.started":"2024-01-08T19:46:17.201153Z","shell.execute_reply":"2024-01-08T19:46:19.005590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[*IMAGE_SIZE, 3]","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:46:19.007898Z","iopub.execute_input":"2024-01-08T19:46:19.008303Z","iopub.status.idle":"2024-01-08T19:46:19.013298Z","shell.execute_reply.started":"2024-01-08T19:46:19.008270Z","shell.execute_reply":"2024-01-08T19:46:19.012516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"', '.join(tf.keras.applications.__dir__())","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:46:19.014194Z","iopub.execute_input":"2024-01-08T19:46:19.014441Z","iopub.status.idle":"2024-01-08T19:46:19.083944Z","shell.execute_reply.started":"2024-01-08T19:46:19.014414Z","shell.execute_reply":"2024-01-08T19:46:19.083301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model weights are saved at the end of every epoch, if it's the best seen so far during model.fit\ncheckpoint_filepath = \"Petals_to_the_Metal-70K_images-trainable_True-MobileNetV2.h5\" #\"Petals_to_the_Metal-70K_images-trainable_True-DenseNet201.h5\"\n\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(\n filepath=checkpoint_filepath,\n save_weights_only=True,\n monitor='val_loss',\n mode='min',\n save_best_only=True\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:46:19.084858Z","iopub.execute_input":"2024-01-08T19:46:19.085135Z","iopub.status.idle":"2024-01-08T19:46:19.090379Z","shell.execute_reply.started":"2024-01-08T19:46:19.085090Z","shell.execute_reply":"2024-01-08T19:46:19.089768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This callback will stop the training when there is no improvement in the validation loss for three consecutive epochs. \nearly_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:46:19.091195Z","iopub.execute_input":"2024-01-08T19:46:19.091453Z","iopub.status.idle":"2024-01-08T19:46:19.100444Z","shell.execute_reply.started":"2024-01-08T19:46:19.091427Z","shell.execute_reply":"2024-01-08T19:46:19.099828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NotFoundError = \"\"\"\nclass LRTensorBoard(TensorBoard):\n def __init__(self, log_dir, **kwargs): # add other arguments to __init__ if you need\n super().__init__(log_dir=log_dir, **kwargs)\n\n def on_epoch_end(self, epoch, logs=None):\n logs = logs or {}\n logs.update({'lr': K.eval(self.model.optimizer.lr)})\n super().on_epoch_end(epoch, logs)\n\nlr_tracking = LRTensorBoard(log_dir=\"./lr_tracking\")\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:46:19.101282Z","iopub.execute_input":"2024-01-08T19:46:19.101509Z","iopub.status.idle":"2024-01-08T19:46:19.114619Z","shell.execute_reply.started":"2024-01-08T19:46:19.101486Z","shell.execute_reply":"2024-01-08T19:46:19.113734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LearningRateTracking(tf.keras.callbacks.Callback):\n def on_epoch_end(self, epoch, logs=None):\n keys = list(logs.keys())\n print(\"End epoch {} of training; got log keys: {}\".format(epoch, keys))\n \n #logs = logs or {}\n #logs.update({'lr': K.eval(self.model.optimizer.lr)}) #optimizer._decayed_lr('float32').numpy()\n #return \n\n#lr_tracking = LearningRateTracking()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:46:19.115439Z","iopub.execute_input":"2024-01-08T19:46:19.115678Z","iopub.status.idle":"2024-01-08T19:46:19.124393Z","shell.execute_reply.started":"2024-01-08T19:46:19.115652Z","shell.execute_reply":"2024-01-08T19:46:19.123678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_efficientnet = False #tuning9\nif use_efficientnet:\n !pip install -q efficientnet\n from efficientnet.tfkeras import EfficientNetB7","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:47:20.955966Z","iopub.execute_input":"2024-01-08T19:47:20.956310Z","iopub.status.idle":"2024-01-08T19:47:20.960828Z","shell.execute_reply.started":"2024-01-08T19:47:20.956283Z","shell.execute_reply":"2024-01-08T19:47:20.959951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weight_per_class = True\n\nif weight_per_class:\n from collections import Counter\n import gc\n\n gc.enable()\n\n def get_training_dataset_raw():\n dataset = load_dataset(TRAINING_FILENAMES, labeled = True, ordered = False)\n return dataset\n\n raw_training_dataset = get_training_dataset_raw()\n\n label_counter = Counter()\n for images, labels in raw_training_dataset:\n label_counter.update([labels.numpy()])\n print(label_counter)\n del raw_training_dataset \n\n TARGET_NUM_PER_CLASS = 122 #?\n\n def get_weight_for_class(class_id):\n counting = label_counter[class_id]\n weight = TARGET_NUM_PER_CLASS / counting\n return weight\n\n weight_per_class = {class_id: get_weight_for_class(class_id) for class_id in range(104)}\n print(weight_per_class)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:51:43.918356Z","iopub.execute_input":"2024-01-08T19:51:43.918668Z","iopub.status.idle":"2024-01-08T19:51:47.177471Z","shell.execute_reply.started":"2024-01-08T19:51:43.918642Z","shell.execute_reply":"2024-01-08T19:51:47.176461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if weight_per_class:\n data = pd.DataFrame.from_dict(weight_per_class, orient='index', columns=['class_weight'])\n plt.figure(figsize=(30, 9))\n\n #barplot color based on value\n bplot = sns.barplot(x=data.index, y='class_weight', data=data, palette= cm.Blues(data['class_weight']*0.15));\n for p in bplot.patches:\n bplot.annotate(format(p.get_height(), '.1f'), \n (p.get_x() + p.get_width() / 2., p.get_height()), \n ha = 'center', va = 'center', \n xytext = (0, 9), \n textcoords = 'offset points')\n plt.xlabel(\"Class\", size=14)\n plt.ylabel(\"Class weight (inverse of %)\", size=14)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:47:28.308013Z","iopub.execute_input":"2024-01-08T19:47:28.308355Z","iopub.status.idle":"2024-01-08T19:47:29.926384Z","shell.execute_reply.started":"2024-01-08T19:47:28.308322Z","shell.execute_reply":"2024-01-08T19:47:29.925233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"using_ensemble_models = False","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:57:45.799172Z","iopub.execute_input":"2024-01-08T19:57:45.799507Z","iopub.status.idle":"2024-01-08T19:57:45.803517Z","shell.execute_reply.started":"2024-01-08T19:57:45.799471Z","shell.execute_reply":"2024-01-08T19:57:45.802621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not using_ensemble_models:\n with strategy.scope():\n #pretrained_model = tf.keras.applications.VGG16\n #pretrained_model = tf.keras.applications.DenseNet201\n #pretrained_model = tf.keras.applications.InceptionResNetV2\n #pretrained_model = tf.keras.applications.InceptionV3\n #pretrained_model = tf.keras.applications.MobileNet\n #pretrained_model = tf.keras.applications.MobileNetV2\n #pretrained_model = tf.keras.applications.NASNetMobile\n #pretrained_model = tf.keras.applications.ResNet50\n #pretrained_model = tf.keras.applications.ResNet101V2\n #pretrained_model = tf.keras.applications.VGG19\n #pretrained_model = tf.keras.applications.Xception\n #pretrained_model = tf.keras.applications.DenseNet201 \n #pretrained_model = EfficientNetB7\n\n pretrained_model = tf.keras.applications.MobileNetV2(\n include_top=False ,\n weights=None, #tuning10 weights='noisy-student' instead of 'imagenet'\n #Self-training with Noisy Student improves ImageNet classification https://arxiv.org/abs/1911.04252) \n #pooling='avg', #tuning1\n input_shape=[*IMAGE_SIZE, 3]\n )\n\n pretrained_model.trainable = True #tuning8 pretrained_model.trainable = True\n\n model = tf.keras.Sequential([\n pretrained_model, #Base pretrained on ImageNet to extract features from images\n\n tf.keras.layers.GlobalAveragePooling2D(), ##Attach a new head to act as a classifier\n #tf.keras.layers.Dropout(0.3), #tuning3\n tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n ])","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:57:46.544779Z","iopub.execute_input":"2024-01-08T19:57:46.545418Z","iopub.status.idle":"2024-01-08T19:57:55.883455Z","shell.execute_reply.started":"2024-01-08T19:57:46.545383Z","shell.execute_reply":"2024-01-08T19:57:55.882490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not using_ensemble_models:\n model.compile(\n optimizer='nadam', #tuning2 optimizer='nadam',\n loss = 'sparse_categorical_crossentropy',\n metrics=['sparse_categorical_accuracy'],\n )","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:58:12.627668Z","iopub.execute_input":"2024-01-08T19:58:12.628043Z","iopub.status.idle":"2024-01-08T19:58:12.862470Z","shell.execute_reply.started":"2024-01-08T19:58:12.628006Z","shell.execute_reply":"2024-01-08T19:58:12.861621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not using_ensemble_models:\n model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:58:13.790810Z","iopub.execute_input":"2024-01-08T19:58:13.791143Z","iopub.status.idle":"2024-01-08T19:58:13.816881Z","shell.execute_reply.started":"2024-01-08T19:58:13.791113Z","shell.execute_reply":"2024-01-08T19:58:13.816079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#if not using_ensemble_models:\ntf.keras.utils.plot_model(model, show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:58:16.110451Z","iopub.execute_input":"2024-01-08T19:58:16.110777Z","iopub.status.idle":"2024-01-08T19:58:16.115363Z","shell.execute_reply.started":"2024-01-08T19:58:16.110750Z","shell.execute_reply":"2024-01-08T19:58:16.114518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not using_ensemble_models:\n # Define training epochs\n EPOCHS = 30\n\n # Define the batch size. This will be 16 with TPU off and 128 (=16*8) with TPU on\n BATCH_SIZE = 16 * strategy.num_replicas_in_sync #See Note 3.1 above 😀\n \n STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:58:17.566782Z","iopub.execute_input":"2024-01-08T19:58:17.567113Z","iopub.status.idle":"2024-01-08T19:58:17.571416Z","shell.execute_reply.started":"2024-01-08T19:58:17.567072Z","shell.execute_reply":"2024-01-08T19:58:17.570583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not using_ensemble_models:\n # Learning Rate Schedule for Fine Tuning #\n def exponential_lr(epoch,\n start_lr = 0.00001, min_lr = 0.00001, max_lr = 0.00005 * strategy.num_replicas_in_sync, #tuning1\n rampup_epochs = 5, sustain_epochs = 0,\n exp_decay = 0.75): #tuning1\n\n def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs, sustain_epochs, exp_decay):\n # linear increase from start to rampup_epochs\n if epoch < rampup_epochs:\n lr = ((max_lr - start_lr) /\n rampup_epochs * epoch + start_lr)\n # constant max_lr during sustain_epochs\n elif epoch < rampup_epochs + sustain_epochs:\n lr = max_lr\n # exponential decay towards min_lr\n else:\n lr = ((max_lr - min_lr) *\n exp_decay**(epoch - rampup_epochs - sustain_epochs) +\n min_lr)\n return lr\n return lr(epoch,\n start_lr,\n min_lr,\n max_lr,\n rampup_epochs,\n sustain_epochs,\n exp_decay)\n\n lr_callback = tf.keras.callbacks.LearningRateScheduler(exponential_lr, verbose=True)\n\n rng = [i for i in range(EPOCHS)]\n y = [exponential_lr(x) for x in rng]\n plt.plot(rng, y)\n print(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:58:18.872396Z","iopub.execute_input":"2024-01-08T19:58:18.872738Z","iopub.status.idle":"2024-01-08T19:58:19.058246Z","shell.execute_reply.started":"2024-01-08T19:58:18.872708Z","shell.execute_reply":"2024-01-08T19:58:19.057367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not using_ensemble_models:\n history = model.fit(\n ds_train,\n validation_data=ds_valid,\n epochs=EPOCHS,\n steps_per_epoch=STEPS_PER_EPOCH,\n callbacks=[lr_callback, checkpoint], # Model weights are saved at the end of every epoch, if it's the best seen so far\n #workers = 3 #tuning5 https://www.tensorflow.org/tutorials/distribute/multi_worker_with_keras\n class_weight = weight_per_class #tuning11\n )","metadata":{"execution":{"iopub.status.busy":"2024-01-08T19:58:20.337528Z","iopub.execute_input":"2024-01-08T19:58:20.337842Z","iopub.status.idle":"2024-01-08T20:34:31.148963Z","shell.execute_reply.started":"2024-01-08T19:58:20.337816Z","shell.execute_reply":"2024-01-08T20:34:31.147627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not using_ensemble_models:\n display_training_curves_v2( \n history.history['loss'],\n history.history['val_loss'],\n history.history['lr'],\n 'loss',\n 211,\n )\n\n display_training_curves_v2(\n history.history['sparse_categorical_accuracy'],\n history.history['val_sparse_categorical_accuracy'],\n history.history['lr'],\n 'accuracy',\n 212,\n )","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:34:31.151688Z","iopub.execute_input":"2024-01-08T20:34:31.151990Z","iopub.status.idle":"2024-01-08T20:34:31.882131Z","shell.execute_reply.started":"2024-01-08T20:34:31.151959Z","shell.execute_reply":"2024-01-08T20:34:31.880999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zoom_after = 20\nif not using_ensemble_models:\n display_training_curves(\n history.history['loss'][zoom_after:],\n history.history['val_loss'][zoom_after:],\n 'loss',\n 211,\n )\n\n display_training_curves(\n history.history['sparse_categorical_accuracy'][zoom_after:],\n history.history['val_sparse_categorical_accuracy'][zoom_after:],\n 'accuracy',\n 212,\n )","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:34:31.883133Z","iopub.execute_input":"2024-01-08T20:34:31.883386Z","iopub.status.idle":"2024-01-08T20:34:32.371346Z","shell.execute_reply.started":"2024-01-08T20:34:31.883361Z","shell.execute_reply":"2024-01-08T20:34:32.370353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zoom_after = 20\nif not using_ensemble_models:\n display_training_curves(\n history.history['loss'][zoom_after:],\n history.history['val_loss'][zoom_after:],\n 'loss',\n 211,\n )\n\n display_training_curves(\n history.history['sparse_categorical_accuracy'][zoom_after:],\n history.history['val_sparse_categorical_accuracy'][zoom_after:],\n 'accuracy',\n 212,\n )","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:34:32.373177Z","iopub.execute_input":"2024-01-08T20:34:32.373442Z","iopub.status.idle":"2024-01-08T20:34:32.851982Z","shell.execute_reply.started":"2024-01-08T20:34:32.373414Z","shell.execute_reply":"2024-01-08T20:34:32.850850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint_filepath","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:34:32.853086Z","iopub.execute_input":"2024-01-08T20:34:32.853463Z","iopub.status.idle":"2024-01-08T20:34:32.858492Z","shell.execute_reply.started":"2024-01-08T20:34:32.853435Z","shell.execute_reply":"2024-01-08T20:34:32.857550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not using_ensemble_models:\n model.load_weights(checkpoint_filepath)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:34:32.859595Z","iopub.execute_input":"2024-01-08T20:34:32.860146Z","iopub.status.idle":"2024-01-08T20:34:42.105258Z","shell.execute_reply.started":"2024-01-08T20:34:32.860103Z","shell.execute_reply":"2024-01-08T20:34:42.104117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:34:42.106508Z","iopub.execute_input":"2024-01-08T20:34:42.106782Z","iopub.status.idle":"2024-01-08T20:34:42.132574Z","shell.execute_reply.started":"2024-01-08T20:34:42.106755Z","shell.execute_reply":"2024-01-08T20:34:42.131801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(checkpoint_filepath)\ntflite_model_name = checkpoint_filepath.replace('.h5', '.tflite')\ntflite_model_name","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:34:42.133787Z","iopub.execute_input":"2024-01-08T20:34:42.134056Z","iopub.status.idle":"2024-01-08T20:34:42.139372Z","shell.execute_reply.started":"2024-01-08T20:34:42.134030Z","shell.execute_reply":"2024-01-08T20:34:42.138591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the model\nconverter = tf.lite.TFLiteConverter.from_keras_model(model)\ntflite_model = converter.convert()\n\n# Save the model\nwith open(tflite_model_name, 'wb') as f:\n f.write(tflite_model)\n \nprint('TFLiteConversion completed successfully \\U0001F680') ","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:34:42.140316Z","iopub.execute_input":"2024-01-08T20:34:42.140574Z","iopub.status.idle":"2024-01-08T20:35:11.223357Z","shell.execute_reply.started":"2024-01-08T20:34:42.140546Z","shell.execute_reply":"2024-01-08T20:35:11.222313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pretrained_model(model_name, image_dataset_weights, trainable=True):\n pretrained_model= model_name(\n include_top=False ,\n weights=image_dataset_weights, #tuning10 weights='noisy-student' instead of 'imagenet'\n #Self-training with Noisy Student improves ImageNet classification https://arxiv.org/abs/1911.04252) \n input_shape=[*IMAGE_SIZE, 3]\n )\n\n pretrained_model.trainable = trainable #tuning8 pretrained_model.trainable = True\n \n model = tf.keras.Sequential([\n pretrained_model, \n tf.keras.layers.GlobalAveragePooling2D(), \n tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n ])\n \n return model","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.226048Z","iopub.execute_input":"2024-01-08T20:35:11.226357Z","iopub.status.idle":"2024-01-08T20:35:11.231487Z","shell.execute_reply.started":"2024-01-08T20:35:11.226327Z","shell.execute_reply":"2024-01-08T20:35:11.230726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if using_ensemble_models:\n with strategy.scope():\n model_EB7 = get_pretrained_model(EfficientNetB7, 'noisy-student', trainable=True)\n\n model_EB7.load_weights('../input/models/Petals_to_the_Metal-70K_images-trainable_True-EfficientNetB7.h5') ","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.232430Z","iopub.execute_input":"2024-01-08T20:35:11.232677Z","iopub.status.idle":"2024-01-08T20:35:11.244594Z","shell.execute_reply.started":"2024-01-08T20:35:11.232649Z","shell.execute_reply":"2024-01-08T20:35:11.243936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if using_ensemble_models:\n model_EB7.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.245425Z","iopub.execute_input":"2024-01-08T20:35:11.245656Z","iopub.status.idle":"2024-01-08T20:35:11.254125Z","shell.execute_reply.started":"2024-01-08T20:35:11.245631Z","shell.execute_reply":"2024-01-08T20:35:11.253441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if using_ensemble_models:\n with strategy.scope():\n model_D201 = get_pretrained_model(tf.keras.applications.DenseNet201, 'imagenet', trainable=True)\n\n model_D201.load_weights('../input/models/Petals_to_the_Metal-70K_images-trainable_True-DenseNet201.h5') ","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.255249Z","iopub.execute_input":"2024-01-08T20:35:11.255507Z","iopub.status.idle":"2024-01-08T20:35:11.264294Z","shell.execute_reply.started":"2024-01-08T20:35:11.255482Z","shell.execute_reply":"2024-01-08T20:35:11.263680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if using_ensemble_models:\n model_D201.summary()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.265067Z","iopub.execute_input":"2024-01-08T20:35:11.265311Z","iopub.status.idle":"2024-01-08T20:35:11.273293Z","shell.execute_reply.started":"2024-01-08T20:35:11.265286Z","shell.execute_reply":"2024-01-08T20:35:11.272583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.274147Z","iopub.execute_input":"2024-01-08T20:35:11.274382Z","iopub.status.idle":"2024-01-08T20:35:11.449577Z","shell.execute_reply.started":"2024-01-08T20:35:11.274358Z","shell.execute_reply":"2024-01-08T20:35:11.448792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if using_ensemble_models:\n cmdataset = get_validation_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and labels, order matters.\n images_ds = cmdataset.map(lambda image, label: image)\n labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() # get everything as one batch\n\n m1 = model_EB7.predict(images_ds)\n m2 = model_D201.predict(images_ds)\n\n scores = []\n for alpha in np.linspace(0,1,100):\n cm_probabilities = alpha*m1+(1-alpha)*m2\n cm_predictions = np.argmax(cm_probabilities, axis=-1)\n scores.append(f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro'))\n\n print(\"Correct labels: \", cm_correct_labels.shape, cm_correct_labels)\n print(\"Predicted labels: \", cm_predictions.shape, cm_predictions)\n plt.plot(scores)\n\n best_alpha = np.argmax(scores)/100\n cm_probabilities = best_alpha*m1+(1-best_alpha)*m2\n cm_predictions = np.argmax(cm_probabilities, axis=-1)\n\n #best_alpha = 0.35","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.450496Z","iopub.execute_input":"2024-01-08T20:35:11.452146Z","iopub.status.idle":"2024-01-08T20:35:11.459191Z","shell.execute_reply.started":"2024-01-08T20:35:11.452089Z","shell.execute_reply":"2024-01-08T20:35:11.458519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if using_ensemble_models:\n print(best_alpha, max(scores))","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.460019Z","iopub.execute_input":"2024-01-08T20:35:11.460279Z","iopub.status.idle":"2024-01-08T20:35:11.471044Z","shell.execute_reply.started":"2024-01-08T20:35:11.460254Z","shell.execute_reply":"2024-01-08T20:35:11.470394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if using_ensemble_models:\n test_ds = get_test_dataset(ordered=True)\n #best_alpha = 0.35\n\n print('Computing predictions...')\n test_images_ds = test_ds.map(lambda image, idnum: image)\n probabilities1 = model_EB7.predict(test_images_ds)\n probabilities2 = model_D201.predict(test_images_ds)\n\n probabilities = best_alpha * probabilities1 + (1 - best_alpha) * probabilities2\n\n predictions = np.argmax(probabilities, axis=-1)\n print(predictions)\n\n print('Generating submission.csv file...')\n # Get image ids from test set and convert to unicode\n test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\n # Write the submission file\n np.savetxt(\n 'submission.csv',\n np.rec.fromarrays([test_ids, predictions]),\n fmt=['%s', '%d'],\n delimiter=',',\n header='id,label',\n comments='',\n )\n\n # Look at the first few predictions\n !head submission.csv","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.471970Z","iopub.execute_input":"2024-01-08T20:35:11.472242Z","iopub.status.idle":"2024-01-08T20:35:11.481111Z","shell.execute_reply.started":"2024-01-08T20:35:11.472215Z","shell.execute_reply":"2024-01-08T20:35:11.480437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_confusion_matrix(cmat, score, precision, recall):\n plt.figure(figsize=(25,25))\n ax = plt.gca()\n ax.matshow(cmat, cmap='Reds')\n ax.set_xticks(range(len(CLASSES)))\n ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n ax.set_yticks(range(len(CLASSES)))\n ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n titlestring = \"\"\n if score is not None:\n titlestring += 'f1 = {:.3f} '.format(score)\n if precision is not None:\n titlestring += '\\nprecision = {:.3f} '.format(precision)\n if recall is not None:\n titlestring += '\\nrecall = {:.3f} '.format(recall)\n if len(titlestring) > 0:\n ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n \n if not using_ensemble_models:\n print('Epoch with min loss and max accuracy:', np.argmin(history.history['val_loss']), np.argmax(history.history['val_sparse_categorical_accuracy']))\n print('min loss and max accuracy:', round(min(history.history['val_loss']),2), round(max(history.history['val_sparse_categorical_accuracy']),2))\n\n print(titlestring.replace('\\n', ''))\n plt.show()\n \ndef display_training_curves(training, validation, title, subplot):\n if subplot%10==1: # set up the subplots on the first call\n plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n plt.tight_layout()\n ax = plt.subplot(subplot)\n ax.set_facecolor('#F8F8F8')\n ax.plot(training)\n ax.plot(validation)\n ax.set_title('model '+ title)\n ax.set_ylabel(title)\n #ax.set_ylim(0.28,1.05)\n ax.set_xlabel('epoch')\n ax.legend(['train', 'valid.'])","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.481989Z","iopub.execute_input":"2024-01-08T20:35:11.482248Z","iopub.status.idle":"2024-01-08T20:35:11.494553Z","shell.execute_reply.started":"2024-01-08T20:35:11.482220Z","shell.execute_reply":"2024-01-08T20:35:11.493862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cmdataset = get_validation_dataset(ordered=True)\nimages_ds = cmdataset.map(lambda image, label: image)\nlabels_ds = cmdataset.map(lambda image, label: label).unbatch()\n\ncm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy()\n\nif using_ensemble_models:\n print('using_ensemble_models')\n probabilities1 = model_EB7.predict(images_ds)\n probabilities2 = model_D201.predict(images_ds)\n cm_probabilities = best_alpha * probabilities1 + (1 - best_alpha) * probabilities2\nelse:\n cm_probabilities = model.predict(images_ds)\n \ncm_predictions = np.argmax(cm_probabilities, axis=-1)\n\nlabels = range(len(CLASSES))\ncmat = confusion_matrix(\n cm_correct_labels,\n cm_predictions,\n labels=labels,\n)\ncmat = (cmat.T / cmat.sum(axis=1)).T # normalize","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:11.495445Z","iopub.execute_input":"2024-01-08T20:35:11.495673Z","iopub.status.idle":"2024-01-08T20:35:36.099502Z","shell.execute_reply.started":"2024-01-08T20:35:11.495648Z","shell.execute_reply":"2024-01-08T20:35:36.098486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cmat","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:36.100742Z","iopub.execute_input":"2024-01-08T20:35:36.101065Z","iopub.status.idle":"2024-01-08T20:35:36.107419Z","shell.execute_reply.started":"2024-01-08T20:35:36.101030Z","shell.execute_reply":"2024-01-08T20:35:36.106573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = f1_score(\n cm_correct_labels,\n cm_predictions,\n labels=labels,\n average='macro',\n)\n\nprecision = precision_score(\n cm_correct_labels,\n cm_predictions,\n labels=labels,\n average='macro',\n)\n\nrecall = recall_score(\n cm_correct_labels,\n cm_predictions,\n labels=labels,\n average='macro',\n)\n\ndisplay_confusion_matrix(cmat, score, precision, recall)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:36.108439Z","iopub.execute_input":"2024-01-08T20:35:36.108721Z","iopub.status.idle":"2024-01-08T20:35:38.128982Z","shell.execute_reply.started":"2024-01-08T20:35:36.108691Z","shell.execute_reply":"2024-01-08T20:35:38.128019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_performance_report = pd.DataFrame(columns=['model-family', 'model', 'epochs', 'arg min loss', 'arg max accuracy', \n 'min loss', 'max accuracy', 'f1', 'precision', 'recall'])\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family': 'VGG',\n 'model':'VGG16', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':11,\n 'min loss':3.47,\n 'max accuracy':0.23,\n 'f1':0.123,\n 'precision':0.146,\n 'recall':0.226}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family': 'DenseNet',\n 'model':'DenseNet201', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':10,\n 'min loss':1.31,\n 'max accuracy':0.74,\n 'f1':0.643,\n 'precision':0.761,\n 'recall':0.599}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'InceptionResNet',\n 'model':'InceptionResNetV2', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':11,\n 'min loss':1.57,\n 'max accuracy':0.66,\n 'f1':0.513,\n 'precision':0.640,\n 'recall':0.480}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'Inception', \n 'model':'InceptionV3', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':11,\n 'min loss':1.48,\n 'max accuracy':0.69,\n 'f1':0.581,\n 'precision':0.728,\n 'recall':0.538}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'MobileNet', \n 'model':'MobileNet', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':10,\n 'min loss':1.11,\n 'max accuracy':0.76,\n 'f1':0.717,\n 'precision':0.798,\n 'recall':0.679}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'MobileNet',\n 'model':'MobileNetV2', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':11,\n 'min loss':1.26,\n 'max accuracy':0.72,\n 'f1':0.650,\n 'precision':0.763,\n 'recall':0.606}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'NASNetMobile',\n 'model':'NASNetMobile', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':11,\n 'min loss':2.69,\n 'max accuracy':0.38,\n 'f1':0.224,\n 'precision':0.401,\n 'recall':0.203}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'ResNet',\n 'model':'ResNet50', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':11,\n 'min loss':3.85,\n 'max accuracy':0.12,\n 'f1':0.017,\n 'precision':0.035,\n 'recall':0.025}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'ResNet',\n 'model':'R101V2', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':9,\n 'min loss':0.87,\n 'max accuracy':0.83,\n 'f1':0.775,\n 'precision':0.842,\n 'recall':0.741}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'VGG',\n 'model':'VGG19', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':11,\n 'min loss':3.58,\n 'max accuracy':0.21,\n 'f1':0.031,\n 'precision':0.036,\n 'recall':0.048}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'Xception',\n 'model':'Xception', \n 'epochs':12, \n 'arg min loss':11, \n 'arg max accuracy':11,\n 'min loss':1.43,\n 'max accuracy':0.71,\n 'f1':0.575,\n 'precision':0.712,\n 'recall':0.536}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'ResNet',\n 'model':'R2 30e', \n 'epochs':30, \n 'arg min loss':29, \n 'arg max accuracy':28,\n 'min loss':0.83,\n 'max accuracy':0.83,\n 'f1':0.788,\n 'precision':0.863,\n 'recall':0.753}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'ResNet',\n 'model':'R101V2 1,2,3+OF', \n 'epochs':30, \n 'arg min loss':26, \n 'arg max accuracy':27,\n 'min loss':0.52,\n 'max accuracy':0.88,\n 'f1':0.864,\n 'precision':0.916,\n 'recall':0.842}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'DenseNet',\n 'model':'D 1,2', \n 'epochs':30, \n 'arg min loss':29, \n 'arg max accuracy':29,\n 'min loss':0.92,\n 'max accuracy':0.81,\n 'f1':0.767,\n 'precision':0.833,\n 'recall':0.732}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'DenseNet',\n 'model':'D201 1,2,4', \n 'epochs':30, \n 'arg min loss':29, \n 'arg max accuracy':27,\n 'min loss':0.92,\n 'max accuracy':0.82,\n 'f1':0.772,\n 'precision':0.846,\n 'recall':0.734}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'ResNet',\n 'model':'R101V2 1,2,4', \n 'epochs':30, \n 'arg min loss':29, \n 'arg max accuracy':28,\n 'min loss':0.66,\n 'max accuracy':0.85,\n 'f1':0.829,\n 'precision':0.870,\n 'recall':0.802}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'ResNet',\n 'model':'R101V2 1,2,4,5', \n 'epochs':30, \n 'arg min loss':29, \n 'arg max accuracy':23,\n 'min loss':0.66,\n 'max accuracy':0.86,\n 'f1':0.829,\n 'precision':0.883,\n 'recall':0.802}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'DenseNet',\n 'model':'D 1,8', \n 'epochs':30, \n 'arg min loss':26, \n 'arg max accuracy':28,\n 'min loss':0.23,\n 'max accuracy':0.95,\n 'f1':0.945,\n 'precision':0.950,\n 'recall':0.946}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'ResNet',\n 'model':'R101V2 1,8', \n 'epochs':30, \n 'arg min loss':10, \n 'arg max accuracy':16,\n 'min loss':0.36,\n 'max accuracy':0.92,\n 'f1':0.909,\n 'precision':0.913,\n 'recall':0.911}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'ResNet',\n 'model':'D 1,2,8', \n 'epochs':30, \n 'arg min loss':10, \n 'arg max accuracy':11,\n 'min loss':0.21,\n 'max accuracy':0.95,\n 'f1':0.953,\n 'precision':0.960,\n 'recall':0.950}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'EfficientNet',\n 'model':'EB7 1,2,9,10', \n 'epochs':30, \n 'arg min loss':29, \n 'arg max accuracy':27,\n 'min loss':0.73,\n 'max accuracy':0.84,\n 'f1':0.779,\n 'precision':0.839,\n 'recall':0.755}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'EfficientNet',\n 'model':'EB7 +11', \n 'epochs':30, \n 'arg min loss':29, \n 'arg max accuracy':28,\n 'min loss':1.0,\n 'max accuracy':0.81,\n 'f1':0.775,\n 'precision':0.769,\n 'recall':0.821}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'EfficientNet',\n 'model':'EB7 1,2,8,9,10,11', \n 'epochs':30, \n 'arg min loss':15, \n 'arg max accuracy':18,\n 'min loss':0.25,\n 'max accuracy':0.96,\n 'f1':0.955,\n 'precision':0.950,\n 'recall':0.964}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'ResNet',\n 'model':'D 1,2,8,11', \n 'epochs':30, \n 'arg min loss':24, \n 'arg max accuracy':23,\n 'min loss':0.22,\n 'max accuracy':0.95,\n 'f1':0.956,\n 'precision':0.957,\n 'recall':0.958}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'Ensemble',\n 'model':'Ensemble EB7+D201', \n 'epochs':30, \n 'arg min loss':24, \n 'arg max accuracy':23,\n 'min loss':0.22,\n 'max accuracy':0.95,\n 'f1':0.962,\n 'precision':0.960,\n 'recall':0.966}\n\nextra_columns = ['total params', 'trainable params', 'non-trainable params','training time per epoch (sec)']\nmodel_performance_report[extra_columns] = pd.DataFrame([[np.nan, np.nan, np.nan, np.nan]], index=model_performance_report.index)\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'DenseNet',\n 'model':'D 1,2,6',\n 'total params':18_521_768,\n 'trainable params':199_784,\n 'non-trainable params':18_321_984,\n 'training time per epoch (sec)':114,\n 'epochs':30, \n 'arg min loss':29, \n 'arg max accuracy':29,\n 'min loss':0.71,\n 'max accuracy':0.85,\n 'f1':0.826,\n 'precision':0.791,\n 'recall':0.890}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'DenseNet',\n 'model':'D 1,2,6,12',\n 'total params':18_521_768,\n 'trainable params':199_784,\n 'non-trainable params':18_321_984,\n 'training time per epoch (sec)':114,\n 'epochs':30, \n 'arg min loss':29, \n 'arg max accuracy':29,\n 'min loss':0.71,\n 'max accuracy':0.85,\n 'f1':0.826,\n 'precision':0.791,\n 'recall':0.890}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'DenseNet',\n 'model':'D 1,2,6,8',\n 'total params':18_521_768,\n 'trainable params':18_292_712,\n 'non-trainable params':229_056,\n 'training time per epoch (sec)':274,\n 'epochs':30, \n 'arg min loss':26, \n 'arg max accuracy':28,\n 'min loss':0.22,\n 'max accuracy':0.96,\n 'f1':0.948,\n 'precision':0.942,\n 'recall':0.957}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'DenseNet',\n 'model':'D 1,2,6,8,12',\n 'total params':18_521_768,\n 'trainable params':18_292_712,\n 'non-trainable params':229_056,\n 'training time per epoch (sec)':274,\n 'epochs':30, \n 'arg min loss':26, \n 'arg max accuracy':28,\n 'min loss':0.22,\n 'max accuracy':0.96,\n 'f1':0.948,\n 'precision':0.942,\n 'recall':0.957}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'EfficientNet',\n 'model':'EB7 1,2,6,8,9,10,11', \n 'total params':64_364_024,\n 'trainable params':64_053_304,\n 'non-trainable params':310_720,\n 'training time per epoch (sec)':511, \n 'epochs':30, \n 'arg min loss':20, \n 'arg max accuracy':28,\n 'min loss':0.24,\n 'max accuracy':0.96,\n 'f1':0.956,\n 'precision':0.949,\n 'recall':0.967}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'Ensemble',\n 'model':'Ensemble 6,12 EB7+D201', \n 'total params':82_885_792,\n 'trainable params':82_346_016,\n 'non-trainable params':539_776,\n 'training time per epoch (sec)':785, \n 'epochs':30, \n 'arg min loss':20, \n 'arg max accuracy':28,\n 'min loss':0.24,\n 'max accuracy':0.96,\n 'f1':0.962,\n 'precision':0.956,\n 'recall':0.971}\n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'MobileNet',\n 'model':'MobileNetV2 1,2,6', \n 'total params':2_391_208,\n 'trainable params':133_224,\n 'non-trainable params':2_257_984,\n 'training time per epoch (sec)':79, \n 'epochs':30, \n 'arg min loss':29, \n 'arg max accuracy':26,\n 'min loss':0.83,\n 'max accuracy':0.8,\n 'f1':0.781,\n 'precision':0.752,\n 'recall':0.850} \n\nmodel_performance_report.loc[len(model_performance_report)]={ 'model-family':'MobileNet',\n 'model':'MobileNetV2 1,2,6,8', \n 'total params':2_391_208,\n 'trainable params':2_357_096,\n 'non-trainable params':34_112,\n 'training time per epoch (sec)':102, \n 'epochs':30, \n 'arg min loss':24, \n 'arg max accuracy':27,\n 'min loss':0.27,\n 'max accuracy':0.95,\n 'f1':0.936,\n 'precision':0.929,\n 'recall':0.951}","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:38.130708Z","iopub.execute_input":"2024-01-08T20:35:38.130992Z","iopub.status.idle":"2024-01-08T20:35:38.226089Z","shell.execute_reply.started":"2024-01-08T20:35:38.130964Z","shell.execute_reply":"2024-01-08T20:35:38.225015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_performance_report","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:38.227040Z","iopub.execute_input":"2024-01-08T20:35:38.227319Z","iopub.status.idle":"2024-01-08T20:35:38.259311Z","shell.execute_reply.started":"2024-01-08T20:35:38.227291Z","shell.execute_reply":"2024-01-08T20:35:38.258644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sns.set_theme(style=\"white\")\n\n# Plot miles per gallon against horsepower with other semantics\nwith sns.axes_style(\"whitegrid\", {'grid.linestyle': '--'}):\n myplot = sns.relplot(x=\"model\", y=\"max accuracy\", hue=\"model\", size=\"f1\",\n sizes=(100, 1000), alpha=1, palette=\"pastel\", legend=\"brief\", #, “brief”, “full”, or False\n height=15, data=model_performance_report)\n\n#myplot.fig.set_size_inches(25,15)\n\n#Slighlty rotate the x-axis labels so model names to not overlap\nmyplot.set_xticklabels(rotation=45)\n\n#Add yaxis gridlines\nmyplot.axes[0][0].set_yticks(np.arange(0,1.05,0.05), minor=False)\n\n#For each model, add model name, val accuracy and f1 score\ndf = model_performance_report.copy()\nfor line in range(0,df.shape[0]):\n if df['model'][line] in ['D 1,8', 'D 1,2,8', 'D 1,2,6', 'D 1,2,6,12', 'D 1,2,6,8', 'D 1,2,6,8,12', 'D 1,2,8,11', 'R101V2', 'EB7 1,2,6,8,9,10,11']:\n #print(df['model'][line])\n mytext = str(df['model'][line][0]) #+' '+str(df['max accuracy'][line])+' '+str(df['f1'][line])\n else:\n mytext = str(df['model'][line])+' - acc:'+str(df['max accuracy'][line])+' - f1:'+str(df['f1'][line])\n \n myplot.axes[0,0].text(model_performance_report['model'][line], \n df['max accuracy'][line], \n mytext, \n horizontalalignment='left', \n size='medium', \n color='black', \n weight='normal')\n\n#Add title and rename axes \nmyplot.set(title='Petals to the Metal - Model Performance - y axis:val acc, size:f1 score - Milestones: 12 epochs; 30 epochs; Hyperparameter tuning; End to end training; Ensemble models; 5x data; Ensemble models of 5x models', xlabel='Model', ylabel='Validation Accuracy')\n\n#Add annotation for training from scratch models\nx_location, y_location = 10, 0.97\nmyplot.axes[0][0].annotate('End to end training (tuning8)', xy=(x_location+6, y_location), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='#C4F0EF', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for Transfer Learning with tuning models \nx_location, y_location = 1, 0.85\nmyplot.axes[0][0].annotate('Transfer Learning with tuning for 30 epochs', xy=(x_location+9, y_location), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='#F5B78A', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for transfer learning models\nx_location, y_location = 5, 0.23\nmyplot.axes[0][0].annotate('Transfer Learning for 12 epochs', xy=(x_location-2, y_location-0.03), xytext=(x_location, y_location-0.05),\n arrowprops=dict(facecolor='lightgrey', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for transfer learning models\nx_location, y_location = -1.5, 0.8\nmyplot.axes[0][0].annotate('Transfer Learning for 12 epochs', xy=(x_location+6, y_location), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='#FFFDAE', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for Ensemble EB7+D201\nx_location, y_location = 19, 0.97\nmyplot.axes[0][0].annotate('Ensemble EB7+D201', xy=(x_location+5, y_location-0.0005), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='#CCBDFA', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for 70K (5x) additional data\nx_location, y_location = 22, 0.98\nmyplot.axes[0][0].annotate('Additional data 70K (5x)', xy=(x_location+5, y_location-0.01), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='lightgrey', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for Legend and \nx_location, y_location = 15, 0.1\nmyplot.axes[0][0].annotate('Models sorted chronologically, size:f1 score', xy=(x_location+12, y_location), xytext=(x_location, y_location),\n arrowprops=dict(facecolor='black', shrink=0.05, headwidth=20, width=5))\n\n#Add Tuning Legend\nx_location, y_location, y_delta = 15.1, 0.655, 0.03\nmyplot.axes[0][0].annotate('Tuning Legend, models sorted chronologically', xy=(x_location, y_location), size='x-large')\nmyplot.axes[0][0].annotate('tuning1: pooling=avg, exponential_lr()', xy=(x_location, y_location-y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning2: optimizer=nadam', xy=(x_location, y_location-2*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning3: Dropout(0.3)', xy=(x_location, y_location-3*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning4: data_augment_v2 with random_blockout', xy=(x_location, y_location-4*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning5: workers = 3 Multi-worker training with Keras', xy=(x_location, y_location-5*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning6: additional data', xy=(x_location, y_location-6*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning7: data_augment_v3', xy=(x_location, y_location-7*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning8: pretrained_model.trainable=True', xy=(x_location, y_location-8*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning9: EfficientNetB7', xy=(x_location, y_location-9*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning10: noisy-student', xy=(x_location, y_location-10*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning11: weight_per_class', xy=(x_location, y_location-11*y_delta), size='large');\nmyplot.axes[0][0].annotate('tuning12: Test Time Augmentation TTA', xy=(x_location, y_location-12*y_delta), size='large');","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:38.260240Z","iopub.execute_input":"2024-01-08T20:35:38.260464Z","iopub.status.idle":"2024-01-08T20:35:40.523306Z","shell.execute_reply.started":"2024-01-08T20:35:38.260439Z","shell.execute_reply":"2024-01-08T20:35:40.522354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_performance_report = model_performance_report.sort_values(by='max accuracy')\nmodel_performance_report","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:40.524566Z","iopub.execute_input":"2024-01-08T20:35:40.524879Z","iopub.status.idle":"2024-01-08T20:35:40.557138Z","shell.execute_reply.started":"2024-01-08T20:35:40.524847Z","shell.execute_reply":"2024-01-08T20:35:40.556353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sns.set_theme(style=\"white\")\n\n# Plot miles per gallon against horsepower with other semantics\nwith sns.axes_style(\"whitegrid\", {'grid.linestyle': '--'}):\n myplot = sns.relplot(x=\"model\", y=\"max accuracy\", hue=\"model\", size=\"f1\",\n sizes=(100, 1000), alpha=1, palette=\"pastel\", legend=\"brief\", #, “brief”, “full”, or False\n height=15, data=model_performance_report)\n\n#myplot.fig.set_size_inches(25,15)\n\n#Slighlty rotate the x-axis labels so model names to not overlap\nmyplot.set_xticklabels(rotation=70)\n\n#Add yaxis gridlines\nmyplot.axes[0][0].set_yticks(np.arange(0,1.05,0.05), minor=False)\n\n#For each model, add model name, val accuracy and f1 score\ndf = model_performance_report.copy()\nfor line in range(0,df.shape[0]):\n if df['model'][line] in ['D 1,2', 'D 1,8', 'D 1,2,6', 'D 1,2,6,12', 'D 1,2,6,8', 'D 1,2,6,8,12', 'D 1,2,8', 'D 1,2,8,11', 'R101V2', 'EB7 1,2,8,9,10,11', 'EB7 1,2,6,8,9,10,11', 'Ensemble EB7+D201', 'MobileNetV2 1,2,6,8']:\n #print(df['model'][line])\n mytext = str(df['model'][line][0])#+' '+str(df['max accuracy'][line])+' '+str(df['f1'][line])\n else:\n mytext = str(df['model'][line])+' - acc:'+str(df['max accuracy'][line])+' - f1:'+str(df['f1'][line])\n \n myplot.axes[0,0].text(model_performance_report['model'][line], \n df['max accuracy'][line], \n mytext, \n horizontalalignment='left', \n size='medium', \n color='black', \n weight='normal')\n\n#Add title and rename axes \nmyplot.set(title='Petals to the Metal - Model Performance - y axis:val acc, size:f1 score - Milestones: 12 epochs; 30 epochs; Hyperparameter tuning; End to end training; Ensemble models; 5x data; Ensemble models of 5x models', xlabel='Model', ylabel='Validation Accuracy')\n\n#Add annotation for training from scratch models\nx_location, y_location = 14, 0.93\nmyplot.axes[0][0].annotate('End to end training (tuning8)', xy=(x_location+7, y_location), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='#a0e2a7', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for Transfer Learning with tuning models \nx_location, y_location = 4, 0.85\nmyplot.axes[0][0].annotate('Transfer Learning with tuning for 30 epochs', xy=(x_location+9, y_location), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='#CCBDFA', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for transfer learning models\nx_location, y_location = 5, 0.23\nmyplot.axes[0][0].annotate('Transfer Learning for 12 epochs', xy=(x_location-2, y_location-0.03), xytext=(x_location, y_location-0.05),\n arrowprops=dict(facecolor='lightgrey', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for transfer learning models\nx_location, y_location = 0, 0.75\nmyplot.axes[0][0].annotate('Transfer Learning for 12 epochs', xy=(x_location+7, y_location), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='lightgrey', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for Ensemble EB7+D201\nx_location, y_location = 19, 0.96\nmyplot.axes[0][0].annotate('Ensemble EB7+D201', xy=(x_location+5, y_location-0.001), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='#ccbdfa', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for 70K (5x) additional data\nx_location, y_location = 21.5, 0.975\nmyplot.axes[0][0].annotate('Additional data 70K (5x)', xy=(x_location+5.5, y_location-0.005), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='lightgrey', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for Ensemble 6,12 EB7+D201\nx_location, y_location = 26.5, 0.985\nmyplot.axes[0][0].annotate('Ensemble 6,12 EB7+D201', xy=(x_location+5.5, y_location-0.01), xytext=(x_location, y_location+0.01),\n arrowprops=dict(facecolor='#a0e2a7', shrink=0.05, headwidth=20, width=5))\n\n#Add annotation for Legend and \nx_location, y_location = 15, 0.1\nmyplot.axes[0][0].annotate('Models sorted by val accuracy, size:f1 score', xy=(x_location+12, y_location), xytext=(x_location, y_location),\n arrowprops=dict(facecolor='black', shrink=0.05, headwidth=20, width=5))\n\n#Add Tuning Legend\nx_location, y_location, y_delta = 15.1, 0.655, 0.03\nmyplot.axes[0][0].annotate('Tuning Legend, models sorted by performance', xy=(x_location, y_location), size='x-large')\nmyplot.axes[0][0].annotate('tuning1: pooling=avg, exponential_lr()', xy=(x_location, y_location-y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning2: optimizer=nadam', xy=(x_location, y_location-2*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning3: Dropout(0.3)', xy=(x_location, y_location-3*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning4: data_augment_v2 with random_blockout', xy=(x_location, y_location-4*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning5: workers = 3 Multi-worker training with Keras', xy=(x_location, y_location-5*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning6: additional data', xy=(x_location, y_location-6*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning7: data_augment_v3', xy=(x_location, y_location-7*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning8: pretrained_model.trainable=True', xy=(x_location, y_location-8*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning9: EfficientNetB7', xy=(x_location, y_location-9*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning10: noisy-student', xy=(x_location, y_location-10*y_delta), size='large')\nmyplot.axes[0][0].annotate('tuning11: weight_per_class', xy=(x_location, y_location-11*y_delta), size='large');\nmyplot.axes[0][0].annotate('tuning12: Test Time Augmentation TTA', xy=(x_location, y_location-12*y_delta), size='large');","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:40.558649Z","iopub.execute_input":"2024-01-08T20:35:40.558935Z","iopub.status.idle":"2024-01-08T20:35:42.861048Z","shell.execute_reply.started":"2024-01-08T20:35:40.558906Z","shell.execute_reply":"2024-01-08T20:35:42.860146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_performance_report.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:42.865338Z","iopub.execute_input":"2024-01-08T20:35:42.865651Z","iopub.status.idle":"2024-01-08T20:35:42.879792Z","shell.execute_reply.started":"2024-01-08T20:35:42.865619Z","shell.execute_reply":"2024-01-08T20:35:42.879051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter_3d(model_performance_report, \n title='How early (arg min/max) did a model perform best', symbol='model-family', color='model', \n x='epochs', y='arg min loss', z='arg max accuracy',\n size_max=12, opacity=0.7,\n width=1200, height=700,\n )\n\nfig.update_layout(margin=dict(l=0, r=0, b=0, t=30))\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:42.880693Z","iopub.execute_input":"2024-01-08T20:35:42.880933Z","iopub.status.idle":"2024-01-08T20:35:43.345616Z","shell.execute_reply.started":"2024-01-08T20:35:42.880907Z","shell.execute_reply":"2024-01-08T20:35:43.344660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_performance_report.query('epochs == 12')","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:43.346698Z","iopub.execute_input":"2024-01-08T20:35:43.346964Z","iopub.status.idle":"2024-01-08T20:35:43.366478Z","shell.execute_reply.started":"2024-01-08T20:35:43.346937Z","shell.execute_reply":"2024-01-08T20:35:43.365726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Filter only for 12 epoch models\nmodel_performance_report_filtered = model_performance_report.query('epochs == 12').copy()\n\nfig = px.scatter_3d(model_performance_report_filtered, \n title='12 epoch model performance - loss and accuracy - by model-family', symbol='model-family', color='model', \n x='model-family', y='min loss', z='max accuracy', text='model',\n size_max=12, opacity=0.7,\n width=1200, height=700,\n )\n\nfig.update_layout(margin=dict(l=0, r=0, b=0, t=30))\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:43.367287Z","iopub.execute_input":"2024-01-08T20:35:43.367500Z","iopub.status.idle":"2024-01-08T20:35:43.463973Z","shell.execute_reply.started":"2024-01-08T20:35:43.367475Z","shell.execute_reply":"2024-01-08T20:35:43.463290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_performance_report.query('epochs == 30')","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:43.464909Z","iopub.execute_input":"2024-01-08T20:35:43.465171Z","iopub.status.idle":"2024-01-08T20:35:43.490052Z","shell.execute_reply.started":"2024-01-08T20:35:43.465143Z","shell.execute_reply":"2024-01-08T20:35:43.489329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Filter only for 30 epoch models\nmodel_performance_report_filtered = model_performance_report.query('epochs == 30').copy()\n\nfig = px.scatter_3d(model_performance_report_filtered, \n title='30 epoch model performance - loss and accuracy - by model-family', symbol='model-family', color='model', \n x='model-family', y='min loss', z='max accuracy', text='model',\n size_max=12, opacity=0.7,\n width=1200, height=700,\n )\n\nfig.update_layout(margin=dict(l=0, r=0, b=0, t=30))\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:43.490894Z","iopub.execute_input":"2024-01-08T20:35:43.491136Z","iopub.status.idle":"2024-01-08T20:35:43.609181Z","shell.execute_reply.started":"2024-01-08T20:35:43.491095Z","shell.execute_reply":"2024-01-08T20:35:43.608414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Filter only for 30 epoch models\nmodel_performance_report_filtered = model_performance_report.query('epochs == 30').copy()\n\nfig = px.scatter_3d(model_performance_report_filtered, \n title='30 epoch model performance - f1, precision, recall (color) - by model-family', symbol='model-family', color='recall', \n x='model-family', y='f1', z='precision', text='model',\n size_max=12, opacity=0.7,\n width=1200, height=700,\n )\n\nfig.update_layout(margin=dict(l=0, r=0, b=0, t=30))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:43.610156Z","iopub.execute_input":"2024-01-08T20:35:43.610397Z","iopub.status.idle":"2024-01-08T20:35:43.680687Z","shell.execute_reply.started":"2024-01-08T20:35:43.610371Z","shell.execute_reply":"2024-01-08T20:35:43.680043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Filter only for 30 epoch models\nmodel_performance_report_filtered = model_performance_report.query('epochs == 30').copy()\n\nfig = px.scatter_3d(model_performance_report_filtered, \n title='30 epoch model performance - f1, precision, recall - by model-family', symbol='model-family', color='model', \n x='f1', y='precision', z='recall', text='model',\n size_max=12, opacity=0.7,\n width=1200, height=700,\n )\n\nfig.update_layout(margin=dict(l=0, r=0, b=0, t=30))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:43.681527Z","iopub.execute_input":"2024-01-08T20:35:43.681758Z","iopub.status.idle":"2024-01-08T20:35:43.790235Z","shell.execute_reply.started":"2024-01-08T20:35:43.681728Z","shell.execute_reply":"2024-01-08T20:35:43.789553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = get_validation_dataset()\ndataset = dataset.unbatch().batch(20)\nbatch = iter(dataset)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:43.791048Z","iopub.execute_input":"2024-01-08T20:35:43.791295Z","iopub.status.idle":"2024-01-08T20:35:43.861481Z","shell.execute_reply.started":"2024-01-08T20:35:43.791269Z","shell.execute_reply":"2024-01-08T20:35:43.860659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = next(batch)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:43.862434Z","iopub.execute_input":"2024-01-08T20:35:43.862670Z","iopub.status.idle":"2024-01-08T20:35:43.986831Z","shell.execute_reply.started":"2024-01-08T20:35:43.862646Z","shell.execute_reply":"2024-01-08T20:35:43.985372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if using_ensemble_models:\n probabilities1 = model_EB7.predict(images)\n probabilities2 = model_D201.predict(images)\n probabilities = best_alpha * probabilities1 + (1 - best_alpha) * probabilities2\nelse:\n probabilities = model.predict(images)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:43.988275Z","iopub.execute_input":"2024-01-08T20:35:43.988694Z","iopub.status.idle":"2024-01-08T20:35:50.885000Z","shell.execute_reply.started":"2024-01-08T20:35:43.988652Z","shell.execute_reply":"2024-01-08T20:35:50.883808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = np.argmax(probabilities, axis=-1)\ndisplay_batch_of_images((images, labels), predictions)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:50.887253Z","iopub.execute_input":"2024-01-08T20:35:50.887562Z","iopub.status.idle":"2024-01-08T20:35:53.195219Z","shell.execute_reply.started":"2024-01-08T20:35:50.887530Z","shell.execute_reply":"2024-01-08T20:35:53.193928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mismatches = sum(cm_predictions!=cm_correct_labels)\nprint('Number of mismatches on validation data: {} out of {} or ({:.2%})'.format(mismatches, NUM_VALIDATION_IMAGES, mismatches/NUM_VALIDATION_IMAGES))","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:53.196401Z","iopub.execute_input":"2024-01-08T20:35:53.196722Z","iopub.status.idle":"2024-01-08T20:35:53.202803Z","shell.execute_reply.started":"2024-01-08T20:35:53.196688Z","shell.execute_reply":"2024-01-08T20:35:53.201867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cmdataset = get_validation_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and labels, order matters.\nimages_ds = cmdataset.map(lambda image, label: image)\nlabels_ds = cmdataset.map(lambda image, label: label).unbatch()\ncm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() # get everything as one batch\n\nmismatches_images, mismatches_predictions, mismatches_labels = [], [], []\nmismatches_dataset = tf.data.Dataset.from_tensors([])\nval_batch = iter(cmdataset.unbatch().batch(1))\n\nfor image_index in range(NUM_VALIDATION_IMAGES):\n batch = next(val_batch)\n if cm_predictions[image_index] != cm_correct_labels[image_index]:\n print('Predicted vs Correct labels: {}, {}'.format(cm_predictions[image_index], cm_correct_labels[image_index]))\n #display_batch_of_images(batch, np.array([cm_predictions[image_index]]))\n #mismatches_dataset = tf.data.Dataset.from_tensors(batch)\n #mismatches_images.append(tf.data.Dataset.from_tensors(batch))\n #mismatches_predictions.append(cm_predictions[image_index])\n #mismatches_labels.append(cm_correct_labels[image_index])","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:53.203775Z","iopub.execute_input":"2024-01-08T20:35:53.204007Z","iopub.status.idle":"2024-01-08T20:35:59.897253Z","shell.execute_reply.started":"2024-01-08T20:35:53.203982Z","shell.execute_reply":"2024-01-08T20:35:59.896161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = get_validation_dataset()\ndataset = dataset.unbatch().batch(20)\nbatch = iter(dataset)\nimages, labels = next(batch)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:35:59.898418Z","iopub.execute_input":"2024-01-08T20:35:59.898712Z","iopub.status.idle":"2024-01-08T20:36:00.124361Z","shell.execute_reply.started":"2024-01-08T20:35:59.898683Z","shell.execute_reply":"2024-01-08T20:36:00.122836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(3):\n display_batch_of_images((images, labels), predictions, display_mismatches_only=True)\n images, labels = next(batch)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:36:00.125935Z","iopub.execute_input":"2024-01-08T20:36:00.126336Z","iopub.status.idle":"2024-01-08T20:36:08.257572Z","shell.execute_reply.started":"2024-01-08T20:36:00.126296Z","shell.execute_reply":"2024-01-08T20:36:08.256477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_batch = next(ds_iter)\ndisplay_batch_of_images(one_batch)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:36:08.258675Z","iopub.execute_input":"2024-01-08T20:36:08.258947Z","iopub.status.idle":"2024-01-08T20:36:10.505943Z","shell.execute_reply.started":"2024-01-08T20:36:08.258912Z","shell.execute_reply":"2024-01-08T20:36:10.504887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"using_tta = False #tuning12\ntta_iterations = 3","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:36:10.507182Z","iopub.execute_input":"2024-01-08T20:36:10.507478Z","iopub.status.idle":"2024-01-08T20:36:10.511340Z","shell.execute_reply.started":"2024-01-08T20:36:10.507448Z","shell.execute_reply":"2024-01-08T20:36:10.510561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if using_tta:\n def get_test_dataset(ordered=False):\n dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n dataset = dataset.map(data_augment, num_parallel_calls=AUTO) #tuning4\n #dataset = dataset.map(data_augment_v2, num_parallel_calls=AUTO) #tuning4 #error in shapes\n #dataset = dataset.map(data_augment_v3, num_parallel_calls=AUTO) #tuning4 0.44 performance :(\n dataset = dataset.batch(BATCH_SIZE)\n dataset = dataset.prefetch(AUTO)\n return dataset","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:36:10.512235Z","iopub.execute_input":"2024-01-08T20:36:10.512468Z","iopub.status.idle":"2024-01-08T20:36:10.523140Z","shell.execute_reply.started":"2024-01-08T20:36:10.512443Z","shell.execute_reply":"2024-01-08T20:36:10.522214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_tta(model, tta_iterations):\n probs = []\n for i in range(tta_iterations):\n print('TTA iteration ', i)\n test_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n test_images_ds = test_ds.map(lambda image, idnum: image)\n \n if using_ensemble_models:\n print('using_ensemble_models')\n probabilities1 = model_EB7.predict(test_images_ds)\n probabilities2 = model_D201.predict(test_images_ds)\n probabilities = best_alpha * probabilities1 + (1 - best_alpha) * probabilities2\n probs.append(probabilities)\n else:\n probs.append(model.predict(test_images_ds,verbose=0))\n \n return probs","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:36:10.523989Z","iopub.execute_input":"2024-01-08T20:36:10.524236Z","iopub.status.idle":"2024-01-08T20:36:10.535160Z","shell.execute_reply.started":"2024-01-08T20:36:10.524210Z","shell.execute_reply":"2024-01-08T20:36:10.534458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\ntest_images_ds = test_ds.map(lambda image, idnum: image)\n\nif using_tta:\n print('Computing predictions using TTA...')\n probabilities = np.mean(predict_tta(model, tta_iterations), axis=0)\nelse:\n print('Computing predictions...')\n probabilities = model.predict(test_images_ds)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:36:10.535876Z","iopub.execute_input":"2024-01-08T20:36:10.536133Z","iopub.status.idle":"2024-01-08T20:36:44.745932Z","shell.execute_reply.started":"2024-01-08T20:36:10.536086Z","shell.execute_reply":"2024-01-08T20:36:44.744814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('using_ensemble_models:', using_ensemble_models)\nprint('Generating submission.csv file...')\n\n# Get image ids from test set and convert to unicode\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\n# Write the submission file\nnp.savetxt(\n 'submission.csv',\n np.rec.fromarrays([test_ids, predictions]),\n fmt=['%s', '%d'],\n delimiter=',',\n header='id,label',\n comments='',\n)\n\n# Look at the first few predictions\n!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2024-01-08T20:36:44.747164Z","iopub.execute_input":"2024-01-08T20:36:44.747494Z","iopub.status.idle":"2024-01-08T20:36:54.780924Z","shell.execute_reply.started":"2024-01-08T20:36:44.747461Z","shell.execute_reply":"2024-01-08T20:36:54.779501Z"},"trusted":true},"execution_count":null,"outputs":[]}]}