{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')\nprint(GCS_DS_PATH)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-11T01:00:37.570393Z","iopub.execute_input":"2022-01-11T01:00:37.571015Z","iopub.status.idle":"2022-01-11T01:00:38.097281Z","shell.execute_reply.started":"2022-01-11T01:00:37.570939Z","shell.execute_reply":"2022-01-11T01:00:38.096237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom functools import partial\nimport matplotlib.pyplot as plt\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n    print(\"Device:\", tpu.master())\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint(\"Number of replicas:\", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:00:40.044023Z","iopub.execute_input":"2022-01-11T01:00:40.044311Z","iopub.status.idle":"2022-01-11T01:00:46.364828Z","shell.execute_reply.started":"2022-01-11T01:00:40.044283Z","shell.execute_reply":"2022-01-11T01:00:46.363915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\nBATCH_SIZE = 64\nIMAGE_SIZE = [224, 224]","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:01:20.346147Z","iopub.execute_input":"2022-01-11T01:01:20.346506Z","iopub.status.idle":"2022-01-11T01:01:20.351278Z","shell.execute_reply.started":"2022-01-11T01:01:20.346467Z","shell.execute_reply":"2022-01-11T01:01:20.350639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + \"/tfrecords-jpeg-224x224/train/*.tfrec\")\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + \"/tfrecords-jpeg-224x224/val/*.tfrec\")\nTEST_FILENAMES = tf.io.gfile.glob(GCS_DS_PATH + \"/tfrecords-jpeg-224x224/test/*.tfrec\")\nprint(\"Train TFRecord Files:\", len(TRAINING_FILENAMES))\nprint(\"val TFRecord Files:\", len(VALIDATION_FILENAMES))\nprint(\"Test TFRecord Files:\", len(TEST_FILENAMES))","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:01:23.434187Z","iopub.execute_input":"2022-01-11T01:01:23.434669Z","iopub.status.idle":"2022-01-11T01:01:23.664650Z","shell.execute_reply.started":"2022-01-11T01:01:23.434637Z","shell.execute_reply":"2022-01-11T01:01:23.663667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32)\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:01:26.422735Z","iopub.execute_input":"2022-01-11T01:01:26.423054Z","iopub.status.idle":"2022-01-11T01:01:26.428943Z","shell.execute_reply.started":"2022-01-11T01:01:26.423024Z","shell.execute_reply":"2022-01-11T01:01:26.428250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_tfrecord(example, train):\n    tfrecord_format = (\n        {\n            \"image\": tf.io.FixedLenFeature([], tf.string),\n            \"class\": tf.io.FixedLenFeature([], tf.int64),\n        }\n        if train\n        else {\"image\": tf.io.FixedLenFeature([], tf.string),}\n    )\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example[\"image\"])\n    if train:\n        label = tf.cast(example[\"class\"], tf.int32)\n        return image, label\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:14:57.658706Z","iopub.execute_input":"2022-01-11T01:14:57.659625Z","iopub.status.idle":"2022-01-11T01:14:57.666286Z","shell.execute_reply.started":"2022-01-11T01:14:57.659574Z","shell.execute_reply":"2022-01-11T01:14:57.665595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataset(filenames, train=True):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False  # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(\n        filenames\n    )  # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(\n        ignore_order\n    )  # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(\n        partial(read_tfrecord, train=train), num_parallel_calls=AUTOTUNE\n    )\n    # returns a dataset of (image, label) pairs if labeled=True or just images if labeled=False\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:15:09.034211Z","iopub.execute_input":"2022-01-11T01:15:09.034751Z","iopub.status.idle":"2022-01-11T01:15:09.041265Z","shell.execute_reply.started":"2022-01-11T01:15:09.034715Z","shell.execute_reply":"2022-01-11T01:15:09.040347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(filenames, train=True):\n    dataset = load_dataset(filenames, train=train)\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.prefetch(buffer_size=AUTOTUNE)\n    dataset = dataset.batch(BATCH_SIZE)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:15:55.832087Z","iopub.execute_input":"2022-01-11T01:15:55.832467Z","iopub.status.idle":"2022-01-11T01:15:55.838225Z","shell.execute_reply.started":"2022-01-11T01:15:55.832429Z","shell.execute_reply":"2022-01-11T01:15:55.837505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = get_dataset(TRAINING_FILENAMES)\nvalidation_dataset = get_dataset(VALIDATION_FILENAMES)\ntest_dataset = get_dataset(TEST_FILENAMES,train=False)\n\nimage_batch, label_batch = next(iter(train_dataset))\n\n\ndef show_batch(image_batch, label_batch):\n    plt.figure(figsize=(10, 10))\n    for n in range(25):\n        ax = plt.subplot(5, 5, n + 1)\n        plt.imshow(image_batch[n] / 255.0)\n        if label_batch[n]:\n            plt.title(\"MALIGNANT\")\n        else:\n            plt.title(\"BENIGN\")\n        plt.axis(\"off\")\n\n\nshow_batch(image_batch.numpy(), label_batch.numpy())","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:16:41.103204Z","iopub.execute_input":"2022-01-11T01:16:41.103541Z","iopub.status.idle":"2022-01-11T01:16:43.187978Z","shell.execute_reply.started":"2022-01-11T01:16:41.103509Z","shell.execute_reply":"2022-01-11T01:16:43.187087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"initial_learning_rate = 0.01\nlr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n    initial_learning_rate, decay_steps=20, decay_rate=0.96, staircase=True\n)\n\ncheckpoint_cb = tf.keras.callbacks.ModelCheckpoint(\n    \"melanoma_model.h5\", save_best_only=True\n)\n\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(\n    patience=10, restore_best_weights=True\n)","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:16:51.400233Z","iopub.execute_input":"2022-01-11T01:16:51.400550Z","iopub.status.idle":"2022-01-11T01:16:51.407603Z","shell.execute_reply.started":"2022-01-11T01:16:51.400517Z","shell.execute_reply":"2022-01-11T01:16:51.406608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_model():\n    base_model = tf.keras.applications.ResNet50(\n        input_shape=(*IMAGE_SIZE, 3), include_top=False\n    )\n\n    base_model.trainable = False\n\n    inputs = tf.keras.layers.Input([*IMAGE_SIZE, 3])\n    x = tf.keras.applications.resnet.preprocess_input(inputs)\n    x = base_model(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(512, activation=\"relu\")(x)\n    x = tf.keras.layers.Dropout(0.2)(x)\n    outputs = tf.keras.layers.Dense(104, activation=\"softmax\")(x)\n\n    model = tf.keras.Model(inputs=inputs, outputs=outputs)\n\n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule),\n        loss=\"sparse_categorical_crossentropy\",\n        metrics=['accuracy'],\n    )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:02:08.617732Z","iopub.execute_input":"2022-01-11T01:02:08.618243Z","iopub.status.idle":"2022-01-11T01:02:08.627182Z","shell.execute_reply.started":"2022-01-11T01:02:08.618196Z","shell.execute_reply":"2022-01-11T01:02:08.626176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = make_model()\n\nhistory = model.fit(\n    train_dataset,\n    epochs=20,\n    validation_data=validation_dataset,\n    callbacks=[checkpoint_cb, early_stopping_cb],\n)","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:02:17.526885Z","iopub.execute_input":"2022-01-11T01:02:17.527209Z","iopub.status.idle":"2022-01-11T01:06:49.113203Z","shell.execute_reply.started":"2022-01-11T01:02:17.527176Z","shell.execute_reply":"2022-01-11T01:06:49.112102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.save(GCS_DS_PATH + \"/model_1\")","metadata":{"execution":{"iopub.status.busy":"2022-01-10T23:39:09.235062Z","iopub.execute_input":"2022-01-10T23:39:09.235461Z","iopub.status.idle":"2022-01-10T23:39:42.829196Z","shell.execute_reply.started":"2022-01-10T23:39:09.235419Z","shell.execute_reply":"2022-01-10T23:39:42.827619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_model2():\n    base_model = tf.keras.applications.Xception(\n        input_shape=(*IMAGE_SIZE, 3), include_top=False\n    )\n\n    base_model.trainable = False\n\n    inputs = tf.keras.layers.Input([*IMAGE_SIZE, 3])\n    x = tf.keras.applications.xception.preprocess_input(inputs)\n    x = base_model(x)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(512, activation=\"relu\")(x)\n    x = tf.keras.layers.Dropout(0.2)(x)\n    outputs = tf.keras.layers.Dense(104, activation=\"softmax\")(x)\n\n    model2 = tf.keras.Model(inputs=inputs, outputs=outputs)\n\n    model2.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule),\n        loss=\"sparse_categorical_crossentropy\",\n        metrics=['accuracy'],\n    )\n\n    return model2","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:33:51.084398Z","iopub.execute_input":"2022-01-11T01:33:51.084874Z","iopub.status.idle":"2022-01-11T01:33:51.094234Z","shell.execute_reply.started":"2022-01-11T01:33:51.084843Z","shell.execute_reply":"2022-01-11T01:33:51.093179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model2 = make_model()\n\nhistory2 = model2.fit(\n    train_dataset,\n    epochs=20,\n    validation_data=validation_dataset,\n    callbacks=[checkpoint_cb, early_stopping_cb],\n)","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:33:55.116132Z","iopub.execute_input":"2022-01-11T01:33:55.116419Z","iopub.status.idle":"2022-01-11T01:38:27.758509Z","shell.execute_reply.started":"2022-01-11T01:33:55.116390Z","shell.execute_reply":"2022-01-11T01:38:27.757234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_batch_predictions(image_batch):\n    plt.figure(figsize=(10, 10))\n    for n in range(25):\n        ax = plt.subplot(5, 5, n + 1)\n        plt.imshow(image_batch[n] / 255.0)\n        img_array = tf.expand_dims(image_batch[n], axis=0)\n        plt.title(np.argmax(model.predict(img_array)[0]))\n        plt.axis(\"off\")\n\n\nimage_batch = next(iter(test_dataset))\n\nshow_batch_predictions(image_batch)","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:32:55.531441Z","iopub.execute_input":"2022-01-11T01:32:55.531831Z","iopub.status.idle":"2022-01-11T01:33:08.775582Z","shell.execute_reply.started":"2022-01-11T01:32:55.531788Z","shell.execute_reply":"2022-01-11T01:33:08.774573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_batch_predictions(image_batch):\n    plt.figure(figsize=(10, 10))\n    for n in range(25):\n        ax = plt.subplot(5, 5, n + 1)\n        plt.imshow(image_batch[n] / 255.0)\n        img_array = tf.expand_dims(image_batch[n], axis=0)\n        plt.title(np.argmax(model2.predict(img_array)[0]))\n        plt.axis(\"off\")\n\n\nimage_batch = next(iter(test_dataset))\n\nshow_batch_predictions(image_batch)","metadata":{"execution":{"iopub.status.busy":"2022-01-11T01:40:27.154802Z","iopub.execute_input":"2022-01-11T01:40:27.155598Z","iopub.status.idle":"2022-01-11T01:40:40.450727Z","shell.execute_reply.started":"2022-01-11T01:40:27.155541Z","shell.execute_reply":"2022-01-11T01:40:40.449877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}