{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!unzip ../input/diabetic-retinopathy-detection/trainLabels.csv.zip\n!unzip ../input/diabetic-retinopathy-detection/sampleSubmission.csv.zip","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:07.656382Z","iopub.execute_input":"2021-08-11T02:47:07.656852Z","iopub.status.idle":"2021-08-11T02:47:09.167891Z","shell.execute_reply.started":"2021-08-11T02:47:07.656755Z","shell.execute_reply":"2021-08-11T02:47:09.16675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U --pre efficientnet >> /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:09.170356Z","iopub.execute_input":"2021-08-11T02:47:09.170664Z","iopub.status.idle":"2021-08-11T02:47:18.461054Z","shell.execute_reply.started":"2021-08-11T02:47:09.170633Z","shell.execute_reply":"2021-08-11T02:47:18.459634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np\nimport tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nimport matplotlib.pyplot as plt\nimport os\nimport re, math\nimport time\nfrom sklearn.utils import class_weight","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:18.463231Z","iopub.execute_input":"2021-08-11T02:47:18.463621Z","iopub.status.idle":"2021-08-11T02:47:25.513641Z","shell.execute_reply.started":"2021-08-11T02:47:18.463584Z","shell.execute_reply":"2021-08-11T02:47:25.512664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"TPU\"\n\nTFREC_DIR = KaggleDatasets().get_gcs_path('diabetic-tfrecords')\n# TFREC_DIR = KaggleDatasets().get_gcs_path('db-stylized-tfrecords')\n# TFREC_DIR = KaggleDatasets().get_gcs_path('db-alpha-07-tfrecords')\n\n\n# number of folds for CV\nFOLDS = 5\n\n# WHICH IMAGE SIZES TO LOAD \n# CHOOSE 128, 192, 256, 384, 512, 768 \n\nIMG_SIZES = 256\n\nIMAGE_SIZE = [IMG_SIZES, IMG_SIZES]\n\n\n# tune it, dependes on Image, size, TPU or GPU\nBATCH_SIZE = 32\n\nEPOCHS = 15\n\n# WHICH EFFICIENTNET TO USE (B?, B0 from B7)\nEFF_NETS = 5\n\n# WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\nWGTS = 1/FOLDS\n\nNUM_CLASSES = 5","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:25.514795Z","iopub.execute_input":"2021-08-11T02:47:25.515091Z","iopub.status.idle":"2021-08-11T02:47:25.847708Z","shell.execute_reply.started":"2021-08-11T02:47:25.515042Z","shell.execute_reply":"2021-08-11T02:47:25.846755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE == \"GPU\":\n    n_gpu = len(tf.config.experimental.list_physical_devices('GPU'))\n    print(\"Num GPUs Available: \", n_gpu)\n    \n    if n_gpu > 1:\n        print(\"Using strategy for multiple GPU\")\n        strategy = tf.distribute.MirroredStrategy()\n    else:\n        print('Standard strategy for GPU...')\n        strategy = tf.distribute.get_strategy()\n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\n\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:25.849316Z","iopub.execute_input":"2021-08-11T02:47:25.84974Z","iopub.status.idle":"2021-08-11T02:47:31.371643Z","shell.execute_reply.started":"2021-08-11T02:47:25.849695Z","shell.execute_reply":"2021-08-11T02:47:31.370589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# adding some data augmentation\ndata_augmentation = tf.keras.Sequential([\n  tf.keras.layers.experimental.preprocessing.RandomFlip('horizontal'),\n  # introduced in TF 2.3\n  tf.keras.layers.experimental.preprocessing.RandomRotation(0.4),\n])","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:31.372946Z","iopub.execute_input":"2021-08-11T02:47:31.373277Z","iopub.status.idle":"2021-08-11T02:47:31.401506Z","shell.execute_reply.started":"2021-08-11T02:47:31.373248Z","shell.execute_reply":"2021-08-11T02:47:31.400494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tlpd = pd.read_csv('./trainLabels.csv')\ntl = {}\nfor i in range(len(tlpd['image'])):\n    tl[tlpd['image'][i]] = tlpd['level'][i]","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:31.402679Z","iopub.execute_input":"2021-08-11T02:47:31.402948Z","iopub.status.idle":"2021-08-11T02:47:32.040098Z","shell.execute_reply.started":"2021-08-11T02:47:31.402921Z","shell.execute_reply":"2021-08-11T02:47:32.039235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# not using metadata (only image, for now)\ndef read_labeled_tfrecord(example, __return_only_label):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        'patient_id' : tf.io.FixedLenFeature([], tf.int64), \n        'side' : tf.io.FixedLenFeature([], tf.int64),\n        'label' : tf.io.FixedLenFeature([], tf.int64)\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    \n    image = decode_image(example['image'])\n    patient_id = example['patient_id']\n    side = example['side']\n    label = example['label']\n    \n#     if q != 0:\n#         q = np.int64(1)\n    \n#     if q == 0:\n#         label = np.array([1, 0, 0, 0, 0])\n#     elif q == 1:\n#         label = np.array([0, 1, 0, 0, 0])\n#     elif q == 2:\n#         label = np.array([0, 0, 1, 0, 0])\n#     elif q == 3:\n#         label = np.array([0, 0, 0, 1, 0])\n#     else:\n#         label = np.array([0, 0, 0, 0, 1])\n    \n    \n    if __return_only_label:\n        return label\n    return image, label\n\n\ndef read_unlabeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        'patient_id' : tf.io.FixedLenFeature([], tf.int64), \n        'side' : tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    \n    image = decode_image(example['image'])\n    patient_id = example['patient_id']\n    side = example['side']\n    \n    return image, patient_id, side\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\n# count # of images in files.. (embedded in file name)\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:32.042842Z","iopub.execute_input":"2021-08-11T02:47:32.043387Z","iopub.status.idle":"2021-08-11T02:47:32.055387Z","shell.execute_reply.started":"2021-08-11T02:47:32.043354Z","shell.execute_reply":"2021-08-11T02:47:32.054626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dataset(filenames, labeled=True, ordered=False, return_only_label=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.cache()\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(lambda example: read_labeled_tfrecord(example, __return_only_label=return_only_label))\n    # returns a dataset of (image, labels) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\n\n\ndef get_training_dataset(filenames, _return_only_label=False):\n    dataset = load_dataset(filenames, labeled=True, return_only_label=_return_only_label)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE*REPLICAS)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(filenames):\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.cache()\n    dataset = dataset.map(read_unlabeled_tfrecord)    \n    dataset = dataset.batch(BATCH_SIZE*REPLICAS)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:32.057211Z","iopub.execute_input":"2021-08-11T02:47:32.057652Z","iopub.status.idle":"2021-08-11T02:47:32.075245Z","shell.execute_reply.started":"2021-08-11T02:47:32.057622Z","shell.execute_reply":"2021-08-11T02:47:32.074467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n    lr_start   = 0.000005\n    lr_max     = 0.00000125 * REPLICAS * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 2\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n    \n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:32.076407Z","iopub.execute_input":"2021-08-11T02:47:32.076788Z","iopub.status.idle":"2021-08-11T02:47:32.092708Z","shell.execute_reply.started":"2021-08-11T02:47:32.07676Z","shell.execute_reply":"2021-08-11T02:47:32.091815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = np.array(pd.read_csv('./trainLabels.csv')['level'])\nw_j = class_weight.compute_class_weight('balanced', classes=np.unique(y_train), y=y_train)\nclass_weights = dict(zip(np.unique(y_train), w_j))\nclass_weights","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:32.09379Z","iopub.execute_input":"2021-08-11T02:47:32.094243Z","iopub.status.idle":"2021-08-11T02:47:32.156157Z","shell.execute_reply.started":"2021-08-11T02:47:32.094213Z","shell.execute_reply":"2021-08-11T02:47:32.155192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# here we define the DNN Model\n\nEFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n\n# EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n#         efn.EfficientNetB4, tf.keras.applications.ResNet50, efn.EfficientNetB6, efn.EfficientNetB7]\n\n# as default it used B0\n\ndef build_model(dim = 256, ef = 0):\n    inp = tf.keras.layers.Input(shape=(*IMAGE_SIZE, 3))\n    \n#     x = data_augmentation(inp)\n    \n#     base = EFNS[ef](input_shape=(*IMAGE_SIZE, 3), weights='noisy-student', include_top = False)\n\n    base = EFNS[ef](input_shape=(*IMAGE_SIZE, 3), weights='imagenet', include_top = False)\n    \n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(NUM_CLASSES, activation='softmax')(x)\n    \n    model = tf.keras.Model(inputs = inp,outputs = x)\n    \n    opt = tf.keras.optimizers.Adam(learning_rate = 0.00005)\n    loss = tf.keras.losses.SparseCategoricalCrossentropy()\n#     loss = tf.keras.losses.BinaryCrossentropy()\n  \n\n    METRICS = [\n        tf.keras.metrics.CategoricalAccuracy(name='accuracy'),\n#         tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n    ]\n\n    model.compile(optimizer = opt, loss = loss, metrics=METRICS)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:32.157398Z","iopub.execute_input":"2021-08-11T02:47:32.157663Z","iopub.status.idle":"2021-08-11T02:47:32.16664Z","shell.execute_reply.started":"2021-08-11T02:47:32.157638Z","shell.execute_reply":"2021-08-11T02:47:32.16589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ds_weights = get_training_dataset(files_train, _return_only_label=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:32.167652Z","iopub.execute_input":"2021-08-11T02:47:32.168137Z","iopub.status.idle":"2021-08-11T02:47:32.183037Z","shell.execute_reply.started":"2021-08-11T02:47:32.168102Z","shell.execute_reply":"2021-08-11T02:47:32.182131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# yy = [x for x in iter(ds_weights)]","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:32.184442Z","iopub.execute_input":"2021-08-11T02:47:32.185008Z","iopub.status.idle":"2021-08-11T02:47:32.194724Z","shell.execute_reply.started":"2021-08-11T02:47:32.184963Z","shell.execute_reply":"2021-08-11T02:47:32.193823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_train = np.array([target.numpy() for target in iter(ds_weights.unbatch())])","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:32.195898Z","iopub.execute_input":"2021-08-11T02:47:32.196196Z","iopub.status.idle":"2021-08-11T02:47:32.205892Z","shell.execute_reply.started":"2021-08-11T02:47:32.196168Z","shell.execute_reply":"2021-08-11T02:47:32.205002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# constant to customize output\nSHOW_FILES = True\nVERBOSE = 1\nPLOT = 1\n\nskf = KFold(n_splits = FOLDS, shuffle = True, random_state=42)\n\n# for others investigations\n# we store all the history\nhistories = []\n\n# these will be split in folds\nnum_total_train_files = len(tf.io.gfile.glob(TFREC_DIR + '/train*.tfrec'))\n\nfiles_test = []\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(15))):\n    \n    tStart = time.time()\n    \n    # display fold info\n    print('#'*60) \n    print('#### FOLD', fold+1)\n    \n    print('#### Image Size %i, EfficientNet B%i, batch_size %i'%\n          (IMG_SIZES, EFF_NETS, BATCH_SIZE*REPLICAS))\n    print('#### Epochs: %i' %(EPOCHS))\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([TFREC_DIR + '/train%.2i*.tfrec'%x for x in idxT])\n    \n#     TD = KaggleDatasets().get_gcs_path('diabetic-tfrecords')\n#     ft = tf.io.gfile.glob([TD + '/train%.2i*.tfrec'%x for x in idxT])\n\n#     print(len(ft))\n#     files_test.append(ft[10])\n#     files_test.append(ft[11])\n\n#     files_test.append(files_train[9])\n    files_test.append(files_train[10])\n    files_test.append(files_train[11])\n    \n    files_train = files_train[0:10]\n    \n#     TD = KaggleDatasets().get_gcs_path('db-alpha-07-tfrecords')\n#     ft = tf.io.gfile.glob([TD + '/train%.2i*.tfrec'%x for x in idxT])\n    \n#     l = np.random.choice(10, size=5, replace=False)\n    \n#     for i1 in l:\n#         files_train[i1] = ft[i1]\n    \n        \n    np.random.shuffle(files_train) \n    print('#'*60)\n    \n    files_valid = tf.io.gfile.glob([TFREC_DIR + '/train%.2i*.tfrec'%x for x in idxV])\n    \n    if SHOW_FILES:\n        print('Number of training images', count_data_items(files_train))\n        print('Number of validation images', count_data_items(files_valid))\n        \n    # BUILD MODEL\n    if DEVICE=='TPU':\n        # to avoid OOM\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n            \n    K.clear_session()\n    with strategy.scope():\n        model = build_model(dim=IMG_SIZES, ef=EFF_NETS)\n        \n    # callback to save best model for each fold\n    sv = tf.keras.callbacks.ModelCheckpoint(\n                'fold-%i.h5'%fold, monitor='val_loss', verbose=1, save_best_only=True,\n                save_weights_only=True, mode='auto', save_freq='epoch')\n    \n    csv_logger = tf.keras.callbacks.CSVLogger('training_retina-%i.log'%fold)\n    \n    y_train = np.array(pd.read_csv('./trainLabels.csv')['level'])\n    w_j = class_weight.compute_class_weight('balanced', classes=np.unique(y_train), y=y_train)\n    class_weights = dict(zip(np.unique(y_train), w_j))\n\n    # TRAIN\n    history = model.fit(\n        get_training_dataset(files_train), \n        epochs=EPOCHS, \n        callbacks = [sv, get_lr_callback(BATCH_SIZE), csv_logger], \n        steps_per_epoch = count_data_items(files_train)/BATCH_SIZE//REPLICAS,\n        validation_data = get_training_dataset(files_valid), \n        validation_steps = count_data_items(files_valid)/BATCH_SIZE//REPLICAS,\n        verbose=VERBOSE,\n#         class_weight = class_weights\n    )\n    \n    # save all histories\n    histories.append(history)\n    \n    tElapsed = round(time.time() - tStart, 1)\n    \n    print(' ')\n    print('Time (sec) elapsed for fold: ', tElapsed)\n    print('...')\n    print('...')","metadata":{"execution":{"iopub.status.busy":"2021-08-11T02:47:32.207332Z","iopub.execute_input":"2021-08-11T02:47:32.207614Z","iopub.status.idle":"2021-08-11T03:37:22.969018Z","shell.execute_reply.started":"2021-08-11T02:47:32.20758Z","shell.execute_reply":"2021-08-11T03:37:22.967367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tensorflow_datasets as tfds\n\n# print(files_test)\n# dataset = load_dataset(files_test, labeled=True, return_only_label=True)\n# dataset = dataset.repeat() # the training dataset must repeat for several epochs\n# dataset = dataset.shuffle(2048)\n# dataset = dataset.batch(BATCH_SIZE*REPLICAS)\n# dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n# arr = list(tfds.as_numpy(dataset))\n# print(arr.shape())","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:37:22.970478Z","iopub.execute_input":"2021-08-11T03:37:22.970768Z","iopub.status.idle":"2021-08-11T03:37:22.975512Z","shell.execute_reply.started":"2021-08-11T03:37:22.970741Z","shell.execute_reply":"2021-08-11T03:37:22.974244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_loss(hist):\n    plt.figure(figsize=(14,6))\n    \n    plt.plot(hist.history['loss'], label='Training loss')\n    plt.plot(hist.history['val_loss'], label='Validation loss')\n    plt.title('Loss fold n. ' + str(fold + 1) )\n    plt.legend(loc='upper right')\n    plt.ylabel('Loss')\n    plt.xlabel('epoch')\n    plt.grid()\n    plt.show();","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:37:22.977371Z","iopub.execute_input":"2021-08-11T03:37:22.977705Z","iopub.status.idle":"2021-08-11T03:37:22.990472Z","shell.execute_reply.started":"2021-08-11T03:37:22.977676Z","shell.execute_reply":"2021-08-11T03:37:22.989272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(FOLDS):\n    plot_loss(histories[fold])","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:37:22.992198Z","iopub.execute_input":"2021-08-11T03:37:22.992721Z","iopub.status.idle":"2021-08-11T03:37:24.022656Z","shell.execute_reply.started":"2021-08-11T03:37:22.992644Z","shell.execute_reply":"2021-08-11T03:37:24.021551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_acc(hist):\n    plt.figure(figsize=(14,6))\n    \n    plt.plot(hist.history['accuracy'], label='Training accuracy')\n    plt.plot(hist.history['val_accuracy'], label='Validation accuracy')\n    plt.title('Accuracy fold n. ' + str(fold + 1) )\n    plt.legend(loc='lower right')\n    plt.ylabel('Acc')\n    plt.xlabel('epoch')\n    plt.grid()\n    plt.show();","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:37:24.02399Z","iopub.execute_input":"2021-08-11T03:37:24.024291Z","iopub.status.idle":"2021-08-11T03:37:24.030445Z","shell.execute_reply.started":"2021-08-11T03:37:24.024263Z","shell.execute_reply":"2021-08-11T03:37:24.029539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in range(FOLDS):\n    plot_acc(histories[fold])","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:37:24.031765Z","iopub.execute_input":"2021-08-11T03:37:24.032021Z","iopub.status.idle":"2021-08-11T03:37:24.990052Z","shell.execute_reply.started":"2021-08-11T03:37:24.031996Z","shell.execute_reply":"2021-08-11T03:37:24.989252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# files_test = tf.io.gfile.glob(KaggleDatasets().get_gcs_path('diabetic-tfrecords') + '/train*.tfrec')[0:1]\n\nnum_total_test_files = count_data_items(files_test)\nprint('Total test files:', num_total_test_files)\npreds = np.zeros((num_total_test_files, NUM_CLASSES))\n\nwi = [1/FOLDS]*FOLDS\n\navg_acc = 0\n\nfor fold in range(FOLDS):\n    model.load_weights('fold-%i.h5'%fold)\n    \n#     test_loss, test_acc = model.evaluate(get_test_dataset(files_test), verbose = 1, batch_size = 4*BATCH_SIZE,\n#                                         steps = num_total_test_files/4*BATCH_SIZE//REPLICAS)\n    \n    out = model.predict(get_test_dataset(files_test), verbose = 1,\n                                        steps = num_total_test_files/BATCH_SIZE/REPLICAS)\n    preds += out * wi[fold]\n#     print('Test accuracy fold n.', fold+1, ': ', round(test_acc, 4))\n#     avg_acc += test_acc * wi[fold]\n\nprint(preds.ndim)\nprint(preds)\nnp.savetxt('preds.txt', preds)\n\nsparse_preds = np.array([np.argmax(x) for x in preds])\n# print('Average accuracy: ', round(avg_acc,4))","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:37:24.991147Z","iopub.execute_input":"2021-08-11T03:37:24.991544Z","iopub.status.idle":"2021-08-11T03:40:10.55657Z","shell.execute_reply.started":"2021-08-11T03:37:24.991508Z","shell.execute_reply":"2021-08-11T03:40:10.555504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_test = get_test_dataset(files_test)\nimg_names = [(img_name.numpy(), side.numpy()) for img, img_name, side in iter(ds_test.unbatch())]","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:40:10.558398Z","iopub.execute_input":"2021-08-11T03:40:10.558813Z","iopub.status.idle":"2021-08-11T03:41:46.112406Z","shell.execute_reply.started":"2021-08-11T03:40:10.558768Z","shell.execute_reply":"2021-08-11T03:41:46.111372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iname_list = []\nfor x in img_names:\n    if x[1]==0:\n        iname_list.append(str(x[0]) + '_left')\n    else:\n        iname_list.append(str(x[0]) + '_right')        ","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:41:46.11413Z","iopub.execute_input":"2021-08-11T03:41:46.114448Z","iopub.status.idle":"2021-08-11T03:41:46.180342Z","shell.execute_reply.started":"2021-08-11T03:41:46.114413Z","shell.execute_reply":"2021-08-11T03:41:46.179419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total = 0.0\ncorrect = 0.0\nfor i in range(len(sparse_preds)):\n    im = iname_list[i]\n    pred = sparse_preds[i]\n    total += 1\n    if tl[im] == pred:\n        correct += 1\n\nprint(correct/total)","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:41:46.181578Z","iopub.execute_input":"2021-08-11T03:41:46.18184Z","iopub.status.idle":"2021-08-11T03:41:46.221524Z","shell.execute_reply.started":"2021-08-11T03:41:46.181815Z","shell.execute_reply":"2021-08-11T03:41:46.220519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import csv\n# with open('submission.csv', 'w', newline='') as file:\n#     writer = csv.writer(file)\n#     writer.writerow([\"image\", \"level\"])\n#     for i in range(len(sparse_preds)):\n#         writer.writerow([iname_list[i], sparse_preds[i]])","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:41:46.224553Z","iopub.execute_input":"2021-08-11T03:41:46.224834Z","iopub.status.idle":"2021-08-11T03:41:46.229049Z","shell.execute_reply.started":"2021-08-11T03:41:46.224805Z","shell.execute_reply":"2021-08-11T03:41:46.227939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.read_csv('submission.csv')\n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:41:46.230463Z","iopub.execute_input":"2021-08-11T03:41:46.230843Z","iopub.status.idle":"2021-08-11T03:41:46.243964Z","shell.execute_reply.started":"2021-08-11T03:41:46.230815Z","shell.execute_reply":"2021-08-11T03:41:46.243033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(np.unique(submission.level))\n# plt.hist(submission.level,bins=2)\n# plt.show()\n\n# print(len(submission))","metadata":{"execution":{"iopub.status.busy":"2021-08-11T03:41:46.245579Z","iopub.execute_input":"2021-08-11T03:41:46.246012Z","iopub.status.idle":"2021-08-11T03:41:46.257681Z","shell.execute_reply.started":"2021-08-11T03:41:46.24597Z","shell.execute_reply":"2021-08-11T03:41:46.256481Z"},"trusted":true},"execution_count":null,"outputs":[]}]}