{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport os, re\n\nfrom kaggle_datasets import KaggleDatasets","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NEW on TPU in TensorFlow 24: shorter cross-compatible TPU/GPU/multi-GPU/cluster-GPU detection code\n\ntry: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    #strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"basedir = KaggleDatasets().get_gcs_path('tpu-getting-started')\ntfrecordsdir = os.path.join(basedir, \"tfrecords-jpeg-512x512\")\ntraindir = os.path.join(tfrecordsdir, \"train\")\ntestdir = os.path.join(tfrecordsdir, \"test\")\nvaldir = os.path.join(tfrecordsdir, \"val\")\nsubmission_file = os.path.join(basedir, \"sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = (512, 512)\nIMAGE_SHAPE = IMAGE_SIZE + (3, )\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 12\n\nstart_lr = 0.00001\nmin_lr = 0.00001\nmax_lr = 0.00005 * strategy.num_replicas_in_sync\nrampup_epochs = 5\nsustain_epochs = 0\nexp_decay = .8\n\ndef lrfn(epoch):\n    if epoch < rampup_epochs:\n        return (max_lr - start_lr)/rampup_epochs * epoch + start_lr\n    elif epoch < rampup_epochs + sustain_epochs:\n        return max_lr\n    else:\n        return (max_lr - min_lr) * exp_decay**(epoch-rampup_epochs-sustain_epochs) + min_lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch), verbose=True)\n\nrang = np.arange(EPOCHS)\ny = [lrfn(x) for x in rang]\nplt.plot(rang, y)\nprint('Learning rate per epoch:')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_tfrecord_ds(path):\n    filenames = tf.io.gfile.glob(os.path.join(path, \"*\"))\n    return tf.data.TFRecordDataset(filenames, num_parallel_reads=tf.data.AUTOTUNE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_feature_description_train = {\n    'class': tf.io.FixedLenFeature([], tf.int64),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\nimage_feature_description_test = {\n    'id': tf.io.FixedLenFeature([], tf.string),\n    'image': tf.io.FixedLenFeature([], tf.string),\n}\n\ndef parse_image_train(proto):\n    example = tf.io.parse_single_example(proto, image_feature_description_train)\n    image = tf.image.decode_jpeg(example[\"image\"], channels=3)\n    image = tf.reshape(image, IMAGE_SHAPE) \n    label = example[\"class\"]\n    return image, label\n\ndef parse_image_test(proto):\n    example = tf.io.parse_single_example(proto, image_feature_description_test)\n    image = tf.image.decode_jpeg(example[\"image\"], channels=3)\n    image = tf.reshape(image, IMAGE_SHAPE)\n    return image, example[\"id\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rng = tf.random.Generator.from_seed(123)\n\ndef augment(image, label):\n    seed = rng.make_seeds(2)[0]\n    image = tf.image.stateless_random_crop(\n        image, size=IMAGE_SHAPE, seed=seed\n    )\n    seed = rng.make_seeds(2)[0]\n    image = tf.image.stateless_random_brightness(\n        image, max_delta=0.5, seed=seed\n    )\n    seed = rng.make_seeds(2)[0]\n    image = tf.image.stateless_random_flip_left_right(\n        image, seed=seed\n    )\n    return image, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ignore_order = tf.data.Options()\nignore_order.experimental_deterministic = False\nds_train = get_tfrecord_ds(traindir).with_options(ignore_order).map(parse_image_train, num_parallel_calls=tf.data.AUTOTUNE).map(augment, num_parallel_calls=tf.data.AUTOTUNE)\nds_val = get_tfrecord_ds(valdir).with_options(ignore_order).map(parse_image_train, num_parallel_calls=tf.data.AUTOTUNE)\nds_test = get_tfrecord_ds(testdir).map(parse_image_test, num_parallel_calls=tf.data.AUTOTUNE)\n\nds_train = ds_train.repeat().cache().shuffle(15000).batch(BATCH_SIZE).prefetch(tf.data.AUTOTUNE)\nds_val = ds_val.repeat().batch(BATCH_SIZE).cache().prefetch(tf.data.AUTOTUNE)\nds_test = ds_test.batch(BATCH_SIZE).cache().prefetch(tf.data.AUTOTUNE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_data_items(path):\n    filenames = tf.io.gfile.glob(os.path.join(path, \"*\")) \n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nTRAIN_STEPS = -(-count_data_items(traindir) // BATCH_SIZE)\nVAL_STEPS = -(-count_data_items(valdir) // BATCH_SIZE)\nTEST_STEPS = -(-count_data_items(testdir) // BATCH_SIZE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nfor ds in ds_train.take(1):\n    for i in range(9):\n        plt.subplot(3, 3, i + 1)\n        plt.axis(\"off\")\n        plt.imshow(ds[0][i])\n        plt.title(ds[1][i].numpy())\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    preprocess_input = tf.keras.applications.xception.preprocess_input\n    base_model = tf.keras.applications.Xception(\n        input_shape=IMAGE_SHAPE,\n        include_top=False,\n        weights='imagenet'\n    )\n    base_model.trainable = True\n    model = tf.keras.Sequential([\n        tf.keras.layers.Lambda(lambda x: preprocess_input(x), input_shape=IMAGE_SHAPE),\n        base_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(104)\n    ])\n    model.compile(\n        optimizer=\"adam\",\n        loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n        metrics=['accuracy'],\n        steps_per_execution=32,\n    )\n    model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    ds_train, \n    epochs=EPOCHS, \n    validation_data=ds_val,\n    steps_per_epoch=TRAIN_STEPS,\n    validation_steps=VAL_STEPS,\n    callbacks=[lr_callback],\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs_range = range(EPOCHS)\n\nplt.figure(figsize=(10, 10))\nplt.subplot(2, 1, 1)\nplt.plot(epochs_range, acc, label='Training Accuracy')\nplt.plot(epochs_range, val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(2, 1, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_test_image = ds_test.map(lambda image, idnum: image)\npred = model.predict(ds_test_image, steps=TEST_STEPS)\npred_label = tf.math.argmax(pred, 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_test_id = ds_test.map(lambda image, idnum: idnum).unbatch()\nids = [str(x, \"utf-8\") for x in ds_test_id.as_numpy_iterator()]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(submission_file)\ndf[\"label\"] = pred_label\ndf[\"id\"] = ids\ndf.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}