{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [GCViT: Global Context Vision Transformer](https://arxiv.org/pdf/2206.09959v1.pdf)\n\n\n
\n\n \n
\n\n
\n\n\n* Code: https://github.com/awsaf49/gcvit-tf\n \n \n ","metadata":{}},{"cell_type":"code","source":"!pip install -qU gcvit # stable\n# !pip install -qU git+https://github.com/awsaf49/gcvit-tf # latest but unstable","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:44:00.404644Z","iopub.execute_input":"2022-07-21T03:44:00.405461Z","iopub.status.idle":"2022-07-21T03:44:33.813594Z","shell.execute_reply.started":"2022-07-21T03:44:00.405357Z","shell.execute_reply":"2022-07-21T03:44:33.812686Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport os\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nimport gcvit\n\nprint(\"Tensorflow version: \" + tf.__version__)\nprint(\"GCViT version: \", gcvit.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-21T04:03:14.428894Z","iopub.execute_input":"2022-07-21T04:03:14.429578Z","iopub.status.idle":"2022-07-21T04:03:14.435422Z","shell.execute_reply.started":"2022-07-21T04:03:14.429544Z","shell.execute_reply":"2022-07-21T04:03:14.434530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detect my accelerator","metadata":{}},{"cell_type":"code","source":"try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n tpu = None\n strategy = tf.distribute.get_strategy()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:45:06.844763Z","iopub.execute_input":"2022-07-21T03:45:06.845064Z","iopub.status.idle":"2022-07-21T03:45:13.003320Z","shell.execute_reply.started":"2022-07-21T03:45:06.845032Z","shell.execute_reply":"2022-07-21T03:45:13.002388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get my data path","metadata":{}},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path() # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:45:20.365359Z","iopub.execute_input":"2022-07-21T03:45:20.365626Z","iopub.status.idle":"2022-07-21T03:45:20.804664Z","shell.execute_reply.started":"2022-07-21T03:45:20.365599Z","shell.execute_reply":"2022-07-21T03:45:20.803729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set some parameters","metadata":{}},{"cell_type":"code","source":"DIM = 224\nIMAGE_SIZE = [DIM, DIM] # at this size, a GPU will run out of memory. Use the TPU\nEPOCHS = 20\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\n\nNUM_TRAINING_IMAGES = 12753\nNUM_TEST_IMAGES = 7382\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:45:22.549661Z","iopub.execute_input":"2022-07-21T03:45:22.549943Z","iopub.status.idle":"2022-07-21T03:45:22.556015Z","shell.execute_reply.started":"2022-07-21T03:45:22.549917Z","shell.execute_reply":"2022-07-21T03:45:22.555016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load my data\n\nThis data is loaded from Kaggle and automatically sharded to maximize parallelization.","metadata":{}},{"cell_type":"code","source":"def decode_image(image_data):\n image = tf.image.decode_jpeg(image_data, channels=3)\n image = tf.cast(image, tf.float32) / 255.0 # convert image to floats in [0, 1] range\n image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n return image\n\ndef read_labeled_tfrecord(example):\n LABELED_TFREC_FORMAT = {\n \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n \"class\": tf.io.FixedLenFeature([], tf.int64), # shape [] means single element\n }\n example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n image = decode_image(example['image'])\n label = tf.cast(example['class'], tf.int32)\n return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n UNLABELED_TFREC_FORMAT = {\n \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n \"id\": tf.io.FixedLenFeature([], tf.string), # shape [] means single element\n # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n }\n example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n image = decode_image(example['image'])\n idnum = example['id']\n return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n # Read from TFRecords. For optimal performance, reading from multiple files at once and\n # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n ignore_order = tf.data.Options()\n if not ordered:\n ignore_order.experimental_deterministic = False # disable order, increase speed\n\n dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n return dataset\n\ndef get_training_dataset():\n dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + F'/tfrecords-jpeg-{DIM}x{DIM}/train/*.tfrec'), labeled=True)\n dataset = dataset.repeat() # the training dataset must repeat for several epochs\n dataset = dataset.shuffle(2048)\n dataset = dataset.batch(BATCH_SIZE)\n dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n return dataset\n\ndef get_validation_dataset():\n dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + F'/tfrecords-jpeg-{DIM}x{DIM}/val/*.tfrec'), labeled=True, ordered=False)\n dataset = dataset.batch(BATCH_SIZE)\n dataset = dataset.cache()\n dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n return dataset\n\ndef get_test_dataset(ordered=False):\n dataset = load_dataset(tf.io.gfile.glob(GCS_DS_PATH + f'/tfrecords-jpeg-{DIM}x{DIM}/test/*.tfrec'), labeled=False, ordered=ordered)\n dataset = dataset.batch(BATCH_SIZE)\n dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n return dataset\n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = get_validation_dataset()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:45:24.822662Z","iopub.execute_input":"2022-07-21T03:45:24.823046Z","iopub.status.idle":"2022-07-21T03:45:25.202458Z","shell.execute_reply.started":"2022-07-21T03:45:24.823008Z","shell.execute_reply":"2022-07-21T03:45:25.201576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"# Learning rate schedule for TPU, GPU and CPU.\n# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\n\nLR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n if epoch < LR_RAMPUP_EPOCHS:\n lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n lr = LR_MAX\n else:\n lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n return lr\n \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = False)\n\nrng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y, '-o')\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","metadata":{"execution":{"iopub.status.busy":"2022-07-21T03:45:25.726323Z","iopub.execute_input":"2022-07-21T03:45:25.727336Z","iopub.status.idle":"2022-07-21T03:45:25.990715Z","shell.execute_reply.started":"2022-07-21T03:45:25.727283Z","shell.execute_reply":"2022-07-21T03:45:25.989714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build a model","metadata":{}},{"cell_type":"code","source":"from gcvit import GCViTTiny\n\nwith strategy.scope(): \n model = GCViTTiny(input_shape=(*IMAGE_SIZE,3), pretrain=True)\n model.reset_classifier(num_classes=104, head_act='softmax')\n \nmodel.compile(\n optimizer='adam',\n loss='sparse_categorical_crossentropy',\n metrics=['sparse_categorical_accuracy']\n)\n\nmodel.build_graph().summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:04:30.734843Z","iopub.execute_input":"2022-07-21T04:04:30.735152Z","iopub.status.idle":"2022-07-21T04:04:58.533057Z","shell.execute_reply.started":"2022-07-21T04:04:30.735118Z","shell.execute_reply":"2022-07-21T04:04:58.532060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save Checkpoint","metadata":{}},{"cell_type":"code","source":"ckpt_path = 'model.h5'\nckpt_callback = tf.keras.callbacks.ModelCheckpoint(\n filepath=ckpt_path,\n save_best_only=True,\n save_weights_only=True,\n monitor='val_sparse_categorical_accuracy',\n mode='max',\n verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:16:27.059645Z","iopub.execute_input":"2022-07-21T04:16:27.059947Z","iopub.status.idle":"2022-07-21T04:16:27.066013Z","shell.execute_reply.started":"2022-07-21T04:16:27.059916Z","shell.execute_reply":"2022-07-21T04:16:27.065064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"history = model.fit(training_dataset, \n steps_per_epoch=STEPS_PER_EPOCH, \n epochs=EPOCHS, \n validation_data=validation_dataset,\n callbacks=[lr_callback, ckpt_callback])","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:16:28.971033Z","iopub.execute_input":"2022-07-21T04:16:28.971359Z","iopub.status.idle":"2022-07-21T04:17:42.271036Z","shell.execute_reply.started":"2022-07-21T04:16:28.971324Z","shell.execute_reply":"2022-07-21T04:17:42.269619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# History","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib as mpl\nmpl.rcParams['figure.figsize'] = (10, 5)\ndata = pd.DataFrame(history.history)\nmetrics = ['loss', 'sparse_categorical_accuracy']\nfor metric in metrics:\n data[[f'{metric}',f'val_{metric}']].plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:17:58.239831Z","iopub.execute_input":"2022-07-21T04:17:58.240146Z","iopub.status.idle":"2022-07-21T04:17:58.802841Z","shell.execute_reply.started":"2022-07-21T04:17:58.240116Z","shell.execute_reply":"2022-07-21T04:17:58.802003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Best Model","metadata":{}},{"cell_type":"code","source":"# load best model\nmodel.load_weights(ckpt_path)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:18:08.992258Z","iopub.execute_input":"2022-07-21T04:18:08.992674Z","iopub.status.idle":"2022-07-21T04:18:13.582874Z","shell.execute_reply.started":"2022-07-21T04:18:08.992646Z","shell.execute_reply":"2022-07-21T04:18:13.581265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compute your predictions on the test set!\n\nThis will create a file that can be submitted to the competition.","metadata":{}},{"cell_type":"code","source":"# infer\ntest_ds = get_test_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and ids, order matters.\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds, verbose=1)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n\nprint('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')","metadata":{"execution":{"iopub.status.busy":"2022-07-21T04:18:32.328772Z","iopub.execute_input":"2022-07-21T04:18:32.329059Z","iopub.status.idle":"2022-07-21T04:19:30.220309Z","shell.execute_reply.started":"2022-07-21T04:18:32.329030Z","shell.execute_reply":"2022-07-21T04:19:30.219199Z"},"trusted":true},"execution_count":null,"outputs":[]}]}
![](\"https://raw.githubusercontent.com/awsaf49/gcvit-tf/main/image/lvg_arch.PNG\")
![](\"https://img.shields.io/badge/🤗%20Hugging%20Face-Spaces-yellow.svg\"){kind=icon}
\n