{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import math, re, os\nimport numpy as np\nimport tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\n\nprint(\"Tensorflow version \" + tf.__version__)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:03:50.945712Z","iopub.execute_input":"2022-09-05T07:03:50.946069Z","iopub.status.idle":"2022-09-05T07:03:50.952167Z","shell.execute_reply.started":"2022-09-05T07:03:50.946034Z","shell.execute_reply":"2022-09-05T07:03:50.951204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Detect TPU, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:03:50.963004Z","iopub.execute_input":"2022-09-05T07:03:50.963578Z","iopub.status.idle":"2022-09-05T07:04:01.077322Z","shell.execute_reply.started":"2022-09-05T07:03:50.963543Z","shell.execute_reply":"2022-09-05T07:04:01.076215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nIMG_SIZE = [512,512]\nGCS_PATH = KaggleDatasets().get_gcs_path()\nAUTO = tf.data.AUTOTUNE\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nEPOCHS = 100\nNUM_TRAINING_IMAGES = 12753\nNUM_TEST_IMAGES = 7382\nSTEP_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image,tf.float32)/255.0 # range [0,1]\n    image = tf.reshape(image,[*IMG_SIZE,3])\n    return image\n\ndef read_labeled_tfrecord(example):\n    FORMAT = {\n        'image': tf.io.FixedLenFeature([],tf.string),\n        'class': tf.io.FixedLenFeature([],tf.int64)\n    }\n    example = tf.io.parse_single_example(example,FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'],tf.int32)\n    return image, label\ndef read_unlabeled_tfrecord(example):\n    FORMAT = {\n        'image': tf.io.FixedLenFeature([],tf.string),\n        'id': tf.io.FixedLenFeature([],tf.string)\n    }\n    example = tf.io.parse_single_example(example,FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum\n\ndef data_augment(x,y):\n    x = tf.image.random_flip_left_right(x)\n    x = tf.image.random_flip_up_down(x)\n    return x,y\n\ndef load_ds(files, labeled = True, ordered = False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(files)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else\n                         read_unlabeled_tfrecord,\n                         num_parallel_calls = AUTO\n                         )\n    dataset = dataset.map(data_augment,\n                          num_parallel_calls = AUTO)\n    return dataset\ndef get_train_ds():\n    dataset = load_ds(tf.io.gfile.glob(GCS_PATH + '/tfrecords-jpeg-512x512/train/*.tfrec'))\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\ndef get_val_ds():\n    dataset = load_ds(tf.io.gfile.glob(GCS_PATH + '/tfrecords-jpeg-512x512/val/*.tfrec'))\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\ndef get_test_ds(ordered=True):\n    dataset = load_ds(tf.io.gfile.glob(GCS_PATH + '/tfrecords-jpeg-512x512/test/*.tfrec'),\n                      labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:27:09.073535Z","iopub.execute_input":"2022-09-05T07:27:09.073872Z","iopub.status.idle":"2022-09-05T07:27:09.452411Z","shell.execute_reply.started":"2022-09-05T07:27:09.073838Z","shell.execute_reply":"2022-09-05T07:27:09.450865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = get_train_ds()\nval_ds = get_val_ds()","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:04:01.546937Z","iopub.execute_input":"2022-09-05T07:04:01.547411Z","iopub.status.idle":"2022-09-05T07:04:01.911731Z","shell.execute_reply.started":"2022-09-05T07:04:01.547324Z","shell.execute_reply":"2022-09-05T07:04:01.910486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import ResNet152\nfrom tensorflow.keras import layers,Sequential","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:04:01.914292Z","iopub.execute_input":"2022-09-05T07:04:01.914829Z","iopub.status.idle":"2022-09-05T07:04:01.919611Z","shell.execute_reply.started":"2022-09-05T07:04:01.914794Z","shell.execute_reply":"2022-09-05T07:04:01.918739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lr_function(epoch):\n     start_lr = 1e-5; min_lr = 1e-5; max_lr = 1e-4    \n     rampup_epochs = 5; sustain_epochs = 0; exp_decay = .8    \n  \n     def lr(epoch, start_lr, min_lr, max_lr, rampup_epochs,\n            sustain_epochs, exp_decay):\n         if epoch < rampup_epochs:    \n             lr = ((max_lr - start_lr) / rampup_epochs\n                         * epoch + start_lr)\n         elif epoch < rampup_epochs + sustain_epochs:    \n             lr = max_lr\n         else:    #E\n             lr = ((max_lr - min_lr) *\n                       exp_decay**(epoch - rampup_epochs -\n                                     sustain_epochs) + min_lr)\n         return lr\n  \n     return lr(epoch, start_lr, min_lr, max_lr,\n               rampup_epochs, sustain_epochs, exp_decay)\ncallbacks = [\n     tf.keras.callbacks.EarlyStopping(monitor='val_loss',    \n                                      patience=5,\n                                      restore_best_weights=True),\n     tf.keras.callbacks.LearningRateScheduler(lambda epoch: \\\n                                              lr_function(epoch),    #B\n                                              verbose=True)\n    \n ]\n  ","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:04:01.921085Z","iopub.execute_input":"2022-09-05T07:04:01.921307Z","iopub.status.idle":"2022-09-05T07:04:01.935683Z","shell.execute_reply.started":"2022-09-05T07:04:01.921280Z","shell.execute_reply":"2022-09-05T07:04:01.934879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    base=ResNet152(include_top=False,\n                  weights='imagenet',\n                  input_shape = [*IMG_SIZE,3])\n    base.trainable = True\n    model = Sequential([base,\n                        layers.GlobalAveragePooling2D(),\n                        layers.Dense(104,activation='softmax')])\n    model.compile(optimizer='adam',\n                  loss='sparse_categorical_crossentropy',\n                  metrics = ['sparse_categorical_accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:04:01.936617Z","iopub.execute_input":"2022-09-05T07:04:01.936858Z","iopub.status.idle":"2022-09-05T07:04:13.176375Z","shell.execute_reply.started":"2022-09-05T07:04:01.936831Z","shell.execute_reply":"2022-09-05T07:04:13.175281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(base.layers)","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:04:13.177759Z","iopub.execute_input":"2022-09-05T07:04:13.177993Z","iopub.status.idle":"2022-09-05T07:04:13.185686Z","shell.execute_reply.started":"2022-09-05T07:04:13.177967Z","shell.execute_reply":"2022-09-05T07:04:13.184463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:04:13.187461Z","iopub.execute_input":"2022-09-05T07:04:13.187840Z","iopub.status.idle":"2022-09-05T07:04:13.212944Z","shell.execute_reply.started":"2022-09-05T07:04:13.187799Z","shell.execute_reply":"2022-09-05T07:04:13.211915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    train_ds, \n    validation_data = val_ds,\n    epochs = EPOCHS,\n    steps_per_epoch = STEP_PER_EPOCH,\n    callbacks=callbacks\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:04:13.214284Z","iopub.execute_input":"2022-09-05T07:04:13.214639Z","iopub.status.idle":"2022-09-05T07:19:28.512680Z","shell.execute_reply.started":"2022-09-05T07:04:13.214606Z","shell.execute_reply":"2022-09-05T07:19:28.511756Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:20:11.340721Z","iopub.execute_input":"2022-09-05T07:20:11.341012Z","iopub.status.idle":"2022-09-05T07:20:11.346695Z","shell.execute_reply.started":"2022-09-05T07:20:11.340984Z","shell.execute_reply":"2022-09-05T07:20:11.345437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_training_curves(training, validation, title,subplot):\n    if subplot%10 == 1:\n        plt.subplots(figsize = (10,10), facecolor = '#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['train ', 'valid'])","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:20:13.769712Z","iopub.execute_input":"2022-09-05T07:20:13.770226Z","iopub.status.idle":"2022-09-05T07:20:13.778327Z","shell.execute_reply.started":"2022-09-05T07:20:13.770186Z","shell.execute_reply":"2022-09-05T07:20:13.777334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_training_curves(\n    history.history['loss'],\n    history.history['val_loss'],\n    'loss',\n    211,\n)\n\ndisplay_training_curves(\n    history.history['sparse_categorical_accuracy'],\n    history.history['val_sparse_categorical_accuracy'],\n    'accuracy',\n    212,\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:20:15.499021Z","iopub.execute_input":"2022-09-05T07:20:15.499310Z","iopub.status.idle":"2022-09-05T07:20:16.044192Z","shell.execute_reply.started":"2022-09-05T07:20:15.499281Z","shell.execute_reply":"2022-09-05T07:20:16.042970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = get_test_ds()\ntest_images_ds = test_ds.map(lambda x,y:x)\ntest_ids_ds = test_ds.map(lambda x,y:y).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\nprobs = model.predict(test_images_ds)\npreds = np.argmax(probs,axis =-1)\n\nnp.savetxt('./submission.csv',\n          np.rec.fromarrays([test_ids, preds]),\n          fmt=['%s', '%d'],\n          delimiter=',',\n          header='id,label',\n          comments='',\n)\n!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-09-05T07:30:28.931830Z","iopub.execute_input":"2022-09-05T07:30:28.932572Z","iopub.status.idle":"2022-09-05T07:30:40.644421Z","shell.execute_reply.started":"2022-09-05T07:30:28.932535Z","shell.execute_reply":"2022-09-05T07:30:40.643189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}