{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Colab Tpu Start\n#### Data Prepare\n1. Stratified GroupKFold TFRecords https://www.kaggle.com/shigengtian/stratified-groupkfold-tfrecords\n2. upload tfrecords file to gcp storage\n3. upload train_folds.csv to colab"},{"metadata":{},"cell_type":"markdown","source":"##### auth gcp"},{"metadata":{"id":"8Yi9Iwn6iMa-","outputId":"30d99296-3596-4217-f395-2934f9c05675","trusted":false},"cell_type":"code","source":"from google.colab import auth\nauth.authenticate_user()\nproject_id = 'xxxxxx'\n!gcloud config set project {project_id}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### tpu init"},{"metadata":{"id":"FpvUOuC3j27n","outputId":"04674b53-82dd-45fd-bf5f-1c66fd670047","trusted":false},"cell_type":"code","source":"%tensorflow_version 2.x\nimport tensorflow as tf\nprint(\"Tensorflow version \" + tf.__version__)\n\ntry:\n  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n  # print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nexcept ValueError:\n  raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)","execution_count":null,"outputs":[]},{"metadata":{"id":"vFIMfPmgQa0h","trusted":false},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"id":"BUkqft0HXReW","outputId":"41ac138c-f464-48b6-a60b-70862199762b","trusted":false},"cell_type":"code","source":"df_fold = pd.read_csv(\"train_folds.csv\")\nlen(df_fold[df_fold[\"fold\"]!=0])","execution_count":null,"outputs":[]},{"metadata":{"id":"CtGyOp2OcxCx","trusted":false},"cell_type":"code","source":"gcs_pattern = 'gs://xxxx/ranzcr/0_train*'\nfilenames = tf.io.gfile.glob(gcs_pattern)","execution_count":null,"outputs":[]},{"metadata":{"id":"_7XrQXeqXcMa","outputId":"d6008a1d-1943-4a45-f643-44e57021d51a","trusted":false},"cell_type":"code","source":"filenames","execution_count":null,"outputs":[]},{"metadata":{"id":"LW3xomkKljlI","outputId":"19a74477-5116-4e09-b892-0fcce3fce337","trusted":false},"cell_type":"code","source":"import re\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nEPOCHS = 100\ndevice = \"TPU\"\nbatch_size = 8 * tpu_strategy.num_replicas_in_sync\nIMAGE_SIZE = (600, 600)\nstart_lr = 0.0001\nmin_lr = 0.000001\nmax_lr = 0.00005 * tpu_strategy.num_replicas_in_sync\n# max_lr = 0.00005\nrampup_epochs = 5\nsustain_epochs = 0\nexp_decay = .6\n\nAUTO = tf.data.experimental.AUTOTUNE  # optimize different parts of input loading.\ndf_fold = pd.read_csv(\"train_folds.csv\")\n\n\ntrain_fns = tf.io.gfile.glob('gs://xxx/ranzcr/0_train*') # change path\nvalidation_fns = tf.io.gfile.glob('gs://xxx/ranzcr/0_val*') # change path\n\n\n\ndef parse_tfrecord(example):\n    columns = df_fold.columns\n    features = {}\n    byte_features = ['StudyInstanceUID', 'PatientID', 'images']\n    for column in columns:\n        if column in byte_features:\n            features[column] = tf.io.FixedLenFeature([], tf.string)\n        else:\n            features[column] = tf.io.FixedLenFeature([], tf.int64)\n\n    # print(feature)\n    features[\"images\"] = tf.io.FixedLenFeature([], tf.string)\n    example = tf.io.parse_single_example(example, features)\n    image = tf.image.decode_jpeg(example['images'], channels=3)\n    image = tf.image.resize(image, (IMAGE_SIZE))\n    label = []\n    out_label_column = ['StudyInstanceUID', 'PatientID', 'images', 'fold']\n    for column in columns:\n        if column not in out_label_column:\n            label.append(example[column])\n    label = tf.stack(label)\n    return image, label\n\n\ndef load_dataset(filenames):\n    # Read from TFRecords. For optimal performance, we interleave reads from multiple files.\n    records = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    return records.map(parse_tfrecord, num_parallel_calls=AUTO)\n\ndef load_dataset(filenames):\n  # Read from TFRecords. For optimal performance, we interleave reads from multiple files.\n  records = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n  return records.map(parse_tfrecord, num_parallel_calls=AUTO)\n\ndef get_training_dataset():\n    dataset = load_dataset(train_fns)\n    def data_augment(img, one_hot_class):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_flip_up_down(img)\n        img = tf.image.adjust_brightness(img, 0.1)\n        img = tf.image.random_contrast(img, 0.9, 1)\n        img = tf.image.random_saturation(img, 0.9, 1)\n        # img = tf.image.per_image_standardization(img)\n        return img, one_hot_class\n\n    augmented = dataset.map(data_augment, num_parallel_calls=AUTO)\n\n    # Prefetch the next batch while training (autotune prefetch buffer size).\n    return augmented.repeat().shuffle(10000).batch(batch_size).prefetch(AUTO)\n\n\ndef get_validate_dataset():\n    dataset = load_dataset(validation_fns)\n    def data_augment(img, one_hot_class):\n        return img, one_hot_class\n\n    augmented = dataset.map(data_augment, num_parallel_calls=AUTO)\n    return augmented.batch(batch_size).prefetch(AUTO)\n\n\ntraining_dataset = get_training_dataset()\nvalidation_dataset = get_validate_dataset()\n\ndef get_dataset_iterator(dataset, n_examples):\n    return dataset.unbatch().batch(n_examples).as_numpy_iterator()\n\ndef create_model():\n    pretrained_model = tf.keras.applications.EfficientNetB5(input_shape=[*IMAGE_SIZE, 3], include_top=False, drop_connect_rate=0.2) ## chose a model\n    pretrained_model.trainable = True\n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(11, activation='sigmoid')\n    ])\n    model.compile(\n        optimizer='adam',\n        loss='binary_crossentropy',\n        metrics=[tf.keras.metrics.AUC(multi_label=True)]\n    )\n    return model\n\n\nif device == \"TPU\":\n    with tpu_strategy.scope():  # creating the model in the TPUStrategy scope means we will train the model on the TPU\n        print(\"TPU\")\n        model = create_model()\nelse:\n    print(\"GPU\")\n    model = create_model()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train"},{"metadata":{"id":"rR_CtOfCTDwi","outputId":"3884d7b4-ae6e-44b7-9c4d-db5c67d0b25e","trusted":false},"cell_type":"code","source":"def lrfn(epoch):\n  if epoch < rampup_epochs:\n    return (max_lr - start_lr)/rampup_epochs * epoch + start_lr\n  elif epoch < rampup_epochs + sustain_epochs:\n    return max_lr\n  else:\n    return (max_lr - min_lr) * exp_decay**(epoch-rampup_epochs-sustain_epochs) + min_lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda epoch: lrfn(epoch), verbose=True)\n\nrlr = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss', factor = 0.2, patience = 3, verbose = 0, \n                                min_delta = 1e-4, min_lr = 1e-6, mode = 'min')\n\nckp = tf.keras.callbacks.ModelCheckpoint('/content/efn5_v3.h5',monitor = 'val_loss',\n                      verbose = 0, save_best_only = True, mode = 'min')\n\n\nes = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', min_delta = 1e-6, patience = 3, mode = 'min', \n                    restore_best_weights = False, verbose = 0)\n\nhistory = model.fit(training_dataset,\n                    validation_data=validation_dataset,\n                    steps_per_epoch=24080//batch_size,\n                    epochs=EPOCHS,\n                    callbacks=[rlr, ckp, es]\n                    )\n\nhist_df = pd.DataFrame(history.history) ","execution_count":null,"outputs":[]},{"metadata":{"id":"oLLXawkoJ7mQ","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}