{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"colab=0\nshow_files=0\ntstamp=0\nif colab:\n    from google.colab import drive\n    drive.mount('/content/gdrive')\nif (not colab)&show_files:\n    import os\n    for dirname, _, filenames in os.walk('/kaggle/input'):\n        for filename in filenames:\n            print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet\nimport math\nimport pytz\nimport random\nimport numpy as np\nimport pandas as pd\nimport math, re, os, gc\nimport tensorflow as tf\nfrom pathlib import Path\nfrom datetime import datetime\nfrom scipy.stats import rankdata\nimport efficientnet.tfkeras as efn\nfrom matplotlib import pyplot as plt\nfrom sklearn.utils import class_weight\nfrom sklearn.metrics import roc_auc_score\n\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE\n\nif not colab:\n    from kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\nNAME='EffNB0_512'\nNFOLDS=5\nNBEST=2 # the number of best models to use for predictions\nSEED=311\n\nif colab:\n    PATH=Path('/content/gdrive/My Drive/kaggle/input/siim-isic-melanoma-classification/') \n    train=pd.read_csv(PATH/'train.csv.zip')\nelse:\n    PATH=Path('/kaggle/input/siim-isic-melanoma-classification/')\n    train=pd.read_csv(PATH/'train.csv')\n\ntest=pd.read_csv(PATH/'test.csv')\nsub=pd.read_csv(PATH/'sample_submission.csv')\n\nseed_everything(SEED)\nprint(f\"The shape of the training set is {train.shape}.\")\nprint(f\"The shape of the testing set is {test.shape}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"The columns in `train`:\\n {list(train.columns)}.\\n\")\nprint(f\"The columns in `test`:\\n {list(test.columns)}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment \n    # variable is set. On Kaggle this is always the case.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy() \n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH={}\n\nif colab:\n    GCS_PATH['train']='gs://kds-4169ce1e5a624d4b5bd866480682658f2063b18c5cf995cd3316a7e9'\n    GCS_PATH['test']='gs://kds-147ff1400c195e2f8c7b01492e04fd2769e0305e9ef6c01afc9c5870'\nelse:\n    GCS_PATH['train']=KaggleDatasets().get_gcs_path('siim-512x512-tfrec-q95')\n    GCS_PATH['test']=KaggleDatasets().get_gcs_path('siim-512x512-tfrec-q95-test')\n\nprint(GCS_PATH['train'])\nprint(GCS_PATH['test'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nIMAGE_SIZE = [512, 512] # At this size, a GPU will run out of memory. Use the TPU.\n                          # For GPU training, please select 224 x 224 px image size.\nEPOCHS=20\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync\n\nCLASSES = ['benign', 'malignant']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ALL_TRAIN=tf.io.gfile.glob(GCS_PATH['train'] + '/*.tfrec')\n\nVAL_FNAMES={}\nfor fn in range(1, NFOLDS+1):\n    VAL_FNAMES[f\"fold_{fn}\"]=[path for path in ALL_TRAIN if f\"fold_{fn}\" in path]    \n    print(\"Fold\", f'{fn}:', len(VAL_FNAMES[f'fold_{fn}']), \"elements in total.\")\n    \nTRAIN_FNAMES={f'fold_{i}': list(set(ALL_TRAIN)-set(VAL_FNAMES[f'fold_{i}']))\n              for i in range(1, NFOLDS+1)}\n\nTEST_FNAMES = tf.io.gfile.glob(GCS_PATH['test'] + '/*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(ALL_TRAIN), len(TEST_FNAMES), len(TRAIN_FNAMES), len(VAL_FNAMES)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VAL_FNAMES\nTRAIN_FNAMES\nALL_TRAIN","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, \n    # i.e. test10-687.tfrec = 687 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    \n    return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nN_TRAIN_IMGS = {f'fold_{i}': count_data_items(TRAIN_FNAMES[f'fold_{i}'])\n                for i in range(1, NFOLDS+1)}\n\nN_VAL_IMGS = {f'fold_{i}': count_data_items(VAL_FNAMES[f'fold_{i}'])\n              for i in range(1, NFOLDS+1)}\n\nN_TEST_IMGS = count_data_items(TEST_FNAMES)\n\nSTEPS_PER_EPOCH = {f'fold_{i}': N_TRAIN_IMGS[f'fold_{i}'] // BATCH_SIZE\n                   for i in range(1, NFOLDS+1)}\n\nprint(\"=\"*75)\n\nprint(f\"The number of unlabeled test image is {N_TEST_IMGS}. It is common for all folds.\")\n\nfor i in range(1, NFOLDS+1):\n    print(\"=\"*75)\n    print(f\"Fold {i}: {N_TRAIN_IMGS[f'fold_{i}']} training and {N_VAL_IMGS[f'fold_{i}']} validation images.\")\nprint(\"=\"*75)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    # convert image to floats in [0, 1] range\n    image = tf.cast(image, tf.float32) / 255.0 \n    # explicit size needed for TPU\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    \n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        # shape [] means single element\n        ################################\n        # bytestring features\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"patient_id\": tf.io.FixedLenFeature([], tf.string),\n        \"benign_malignant\": tf.io.FixedLenFeature([], tf.string),\n        # integer features\n        \"age\": tf.io.FixedLenFeature([], tf.int64),\n        \"sex_female\": tf.io.FixedLenFeature([], tf.int64),        \n        \"sex_male\": tf.io.FixedLenFeature([], tf.int64),\n        \"sex_unknown\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_head/neck\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_lower extremity\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_oral/genital\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_palms/soles\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_torso\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_unknown\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_upper extremity\": tf.io.FixedLenFeature([], tf.int64),\n        \"height\": tf.io.FixedLenFeature([], tf.int64),\n        \"width\": tf.io.FixedLenFeature([], tf.int64),\n        \"target\": tf.io.FixedLenFeature([], tf.int64), \n        # float features\n        \"age_scaled\": tf.io.FixedLenFeature([], tf.float32),\n    }\n\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    # image data\n    image = decode_image(example['image']) \n    data={}\n    # bytestring features\n    data['image_name']=image_name=tf.cast(example['image_name'], tf.string)\n    data['patient_id']=tf.cast(example['patient_id'], tf.string)\n    # integer features\n    data['age']=tf.cast(example['age'], tf.int32)\n    data['sex_female']=tf.cast(example['sex_female'], tf.int32)\n    data['sex_male']=tf.cast(example['sex_male'], tf.int32)\n    data['sex_unknown']=tf.cast(example['sex_unknown'], tf.int32)\n    data['site_head/neck']=tf.cast(example['site_head/neck'], tf.int32)\n    data['site_lower extremity']=tf.cast(example['site_lower extremity'], tf.int32)\n    data['site_oral/genital']=tf.cast(example['site_oral/genital'], tf.int32)\n    data['site_palms/soles']=tf.cast(example['site_palms/soles'], tf.int32)\n    data['site_torso']=tf.cast(example['site_torso'], tf.int32)\n    data['site_unknown']=tf.cast(example['site_unknown'], tf.int32)\n    data['site_upper extremity']=tf.cast(example['site_upper extremity'], tf.int32)\n#     data['height']=tf.cast(example['height'], tf.int32)\n#     data['width']=tf.cast(example['width'], tf.int32)\n    # float features\n    data['age_scaled']=tf.cast(example['age_scaled'], tf.float32)\n    # target (integer)\n    label=tf.cast(example['target'], tf.int32)\n     # target (string)\n    label_name=tf.cast(example['benign_malignant'], tf.string)\n\n    return image, label, data, label_name\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        # tf.string means bytestring\n        # shape [] means single element\n        ################################\n        # bytestring features\n        \"image\": tf.io.FixedLenFeature([], tf.string), \n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"patient_id\": tf.io.FixedLenFeature([], tf.string),\n        # integer features\n        \"age\": tf.io.FixedLenFeature([], tf.int64),\n        \"sex_female\": tf.io.FixedLenFeature([], tf.int64),        \n        \"sex_male\": tf.io.FixedLenFeature([], tf.int64),\n        \"sex_unknown\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_head/neck\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_lower extremity\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_oral/genital\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_palms/soles\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_torso\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_unknown\": tf.io.FixedLenFeature([], tf.int64),\n        \"site_upper extremity\": tf.io.FixedLenFeature([], tf.int64),\n        \"height\": tf.io.FixedLenFeature([], tf.int64),\n        \"width\": tf.io.FixedLenFeature([], tf.int64), \n        # float features\n        \"age_scaled\": tf.io.FixedLenFeature([], tf.float32),\n    }\n\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    # image data\n    image = decode_image(example['image']) \n    data={}\n    # bytestring features\n    data['image_name']=image_name=tf.cast(example['image_name'], tf.string)\n    data['patient_id']=tf.cast(example['patient_id'], tf.string)\n    # integer features\n    data['age']=tf.cast(example['age'], tf.int32)\n    data['sex_female']=tf.cast(example['sex_female'], tf.int32)\n    data['sex_male']=tf.cast(example['sex_male'], tf.int32)\n    data['sex_unknown']=tf.cast(example['sex_unknown'], tf.int32)\n    data['site_head/neck']=tf.cast(example['site_head/neck'], tf.int32)\n    data['site_lower extremity']=tf.cast(example['site_lower extremity'], tf.int32)\n    data['site_oral/genital']=tf.cast(example['site_oral/genital'], tf.int32)\n    data['site_palms/soles']=tf.cast(example['site_palms/soles'], tf.int32)\n    data['site_torso']=tf.cast(example['site_torso'], tf.int32)\n    data['site_unknown']=tf.cast(example['site_unknown'], tf.int32)\n    data['site_upper extremity']=tf.cast(example['site_upper extremity'], tf.int32)\n#     data['height']=tf.cast(example['height'], tf.int32)\n#     data['width']=tf.cast(example['width'], tf.int32)\n    # float features\n    data['age_scaled']=tf.cast(example['age_scaled'], tf.float32)\n\n    return image, data\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files \n    # at once and disregarding data order. Order does not matter since we will \n    # be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        # disable order, increase speed\n        ignore_order.experimental_deterministic = False\n\n    # automatically interleaves reads from multiple files\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.with_options(ignore_order)\n    # returns a dataset of (image, label) pairs if labeled=True \n    # or (image, id) pairs if labeled=False\n    dataset = dataset.map(read_labeled_tfrecord if labeled \n                          else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntraining_dataset = load_dataset(TRAIN_FNAMES['fold_2'])\n\nprint(\"Example of the training data:\")\nfor image, label, data, label_name in training_dataset.take(1):\n    print(\"The image batch size:\", image.numpy().shape)\n    print(\"Label:\", label.numpy())\n    print(\"Label name:\", label_name.numpy())\n    print(\"Age:\", data['age'].numpy())\n    print(\"Age (scaled):\", data['age_scaled'].numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nvalidation_dataset = load_dataset(VAL_FNAMES['fold_2'])\n\nprint(\"Examples of the validation data:\")\nfor image, label, data, label_name in validation_dataset.take(1):\n    print(\"The image batch size:\", image.numpy().shape)\n    print(\"Label:\", label.numpy())\n    print(\"Label name:\", label_name.numpy())\n    print(\"Age:\", data['age'].numpy())\n    print(\"Age (scaled):\", data['age_scaled'].numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FNAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTO)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntest_dataset = get_test_dataset()\n\nprint(\"Examples of the test data:\")\nfor image, data in test_dataset.take(1):\n    print(\"The image batch size:\", image.numpy().shape)\n    print(\"Ages, 5 examples:\", data['age'].numpy()[:5])\n    print(\"Age (scaled), 5 examples:\", data['age_scaled'].numpy()[:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.set_printoptions(threshold=15, linewidth=80)\ndef batch_to_numpy_images_and_labels(databatch):\n    if len(databatch)==4:\n        images, labels, _, _ = databatch\n        numpy_images = images.numpy()\n        numpy_labels = labels.numpy()\n    else:\n        images, _ = databatch\n        numpy_images = images.numpy()\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" \n                                if not correct else '', \n                                CLASSES[correct_label] if not correct else ''), correct\ndef display_one_image(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), \n                  color='red' if red else 'black', fontdict={'verticalalignment':'center'}, \n                  pad=int(titlesize/1.5)\n                 )\n    return (subplot[0], subplot[1], subplot[2]+1)\ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does  \n    # not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        # magic formula tested to work from 1x1 to 10x10 images\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3\n        subplot = display_one_image(image, title, subplot, \n                                     not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_dataset = training_dataset.batch(20)\ntrain_batch = iter(training_dataset)\n# run this cell again for next set of images\ndisplay_batch_of_images(next(train_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del training_dataset, train_batch\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_dataset = validation_dataset.batch(20)\nvalidation_batch = iter(validation_dataset)\n# run this cell again for next set of images\ndisplay_batch_of_images(next(validation_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del validation_dataset, validation_batch\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = test_dataset.unbatch().batch(20)\ntest_batch = iter(test_dataset)\n# run this cell again for next set of images\ndisplay_batch_of_images(next(test_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test_dataset, test_batch\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 0.000005#0.00001\nLR_MAX = 0.00000725 * strategy.num_replicas_in_sync\nLR_MIN = 0.000005\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 4\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\n# def lrfn(epoch):\n#     LR_START = 0.00001\n#     LR_MAX = 0.00005\n#     LR_MIN = 0.00001\n#     LR_RAMPUP_EPOCHS = 5\n#     LR_SUSTAIN_EPOCHS = 0\n#     LR_EXP_DECAY = .8\n    \n#     if epoch < LR_RAMPUP_EPOCHS:\n#         lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n#     elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n#         lr = LR_MAX\n#     else:\n#         lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n#     return lr\n\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['target'].values\nclass_weights = class_weight.compute_class_weight(class_weight='balanced',\n                                                  classes=np.unique(train['target'].values),\n                                                  y=train['target'].values,\n                                                 )\n\nclass_weights = {i : class_weights[i] for i in range(len(class_weights))}\n\nprint(class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tab_feats=['age_scaled',\n           'sex_female', \n           'sex_male', \n           'sex_unknown', \n           'site_head/neck', \n           'site_lower extremity', \n           'site_oral/genital',\n           'site_palms/soles',\n           'site_torso',\n           'site_unknown',\n           'site_upper extremity',\n#            'height',\n#            'width',\n          ]\n\nN_TAB_FEATS=len(tab_feats)\n\nprint(f\"The number of tabular features is {N_TAB_FEATS}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\n\ndef get_model():\n    with strategy.scope():\n        pretrained_model = efn.EfficientNetB7(input_shape=(*IMAGE_SIZE, 3),\n                                              weights='imagenet',\n                                              include_top=False\n                                             )\n        # False = transfer learning, True = fine-tuning\n        pretrained_model.trainable = True#False \n\n        inp1 = tf.keras.layers.Input(shape=(*IMAGE_SIZE, 3), name='inp1')\n        inp2 = tf.keras.layers.Input(shape=(N_TAB_FEATS), name='inp2')\n        \n        # BUILD MODEL HERE\n        \n        x=pretrained_model(inp1)\n        x=tf.keras.layers.GlobalAveragePooling2D()(x)\n        x=tf.keras.layers.Dense(256, \n                                kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n                                activation='relu')(x)\n        x=tf.keras.layers.Dense(512, \n                                kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n                                activation='relu')(x)\n        x=tf.keras.layers.Dense(1024, \n                                kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n                                activation='relu')(x)\n        x=tf.keras.layers.Dense(512, kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n                                activation='relu')(x)\n        x=tf.keras.layers.Dense(256, kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n                                activation='relu')(x)\n        y=tf.keras.layers.Dense(100, \n                                kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n                                activation='relu')(inp2)\n        \n        concat=tf.keras.layers.concatenate([y, x])\n        \n        output = tf.keras.layers.Dense(1, activation='sigmoid', name='output')(concat)\n        \n        model = tf.keras.models.Model(inputs=[inp1,inp2], outputs=[output])\n    \n        model.compile(\n        optimizer='adam',\n        loss = 'binary_crossentropy',\n        metrics=[tf.keras.metrics.AUC()],\n        )\n        \n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %time\n\n# def get_model():\n#     with strategy.scope():\n#         pretrained_model = efn.EfficientNetB7(input_shape=(*IMAGE_SIZE, 3),\n#                                               weights='imagenet',\n#                                               include_top=False\n#                                              )\n#         # False = transfer learning, True = fine-tuning\n#         pretrained_model.trainable = True#False \n\n#         inp1 = tf.keras.layers.Input(shape=(*IMAGE_SIZE, 3), name='inp1')\n#         inp2 = tf.keras.layers.Input(shape=(N_TAB_FEATS), name='inp2')\n        \n#         # BUILD MODEL HERE\n        \n#         x=pretrained_model(inp1)\n#         x=tf.keras.layers.GlobalAveragePooling2D()(x)\n#         x=tf.keras.layers.Dense(512, \n#                                 kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n#                                 activation='relu')(x)\n#         x=tf.keras.layers.Dropout(0.2)(x)\n#         x=tf.keras.layers.Dense(256, \n#                                 kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n#                                 activation='relu')(x)\n#         x=tf.keras.layers.Dropout(0.2)(x)\n#         x=tf.keras.layers.Dense(128, \n#                                 kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n#                                 activation='relu')(x)\n#         x=tf.keras.layers.Dropout(0.2)(x)\n#         x=tf.keras.layers.Dense(64, kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n#                                 activation='relu')(x)\n#         x=tf.keras.layers.Dropout(0.2)(x)\n        \n#         y=tf.keras.layers.Dense(100, \n#                                 kernel_regularizer=tf.keras.regularizers.l2(l=0.01),\n#                                 activation='relu')(inp2)\n        \n#         concat=tf.keras.layers.concatenate([y, x])\n        \n#         output = tf.keras.layers.Dense(1, activation='sigmoid', name='output')(concat)\n        \n#         model = tf.keras.models.Model(inputs=[inp1,inp2], outputs=[output])\n    \n#         model.compile(\n#         optimizer='adam',\n#         loss = 'binary_crossentropy',\n#         metrics=[tf.keras.metrics.AUC()],\n#         )\n        \n#         return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodel=get_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del model\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if colab:\n    \n    SAVE_FOLDER=NAME\n    \n    if tstamp:\n        time_zone = pytz.timezone('America/Chicago')\n        current_datetime = datetime.now(time_zone)\n        ts=current_datetime.strftime(\"%m%d%H%M%S\")\n        SAVE_FOLDER+='_'+ts\n        \n    SAVE_FOLDER=PATH/SAVE_FOLDER\n    if not os.path.exists(SAVE_FOLDER):\n        os.mkdir(SAVE_FOLDER)\n\nelse:\n    SAVE_FOLDER=Path('/kaggle/working')\nclass save_best_n(tf.keras.callbacks.Callback):\n    def __init__(self, fn, model):\n        self.fn = fn\n        self.model = model\n\n    def on_epoch_end(self, epoch, logs=None):\n        \n        if (epoch>0):\n            score=logs.get(\"val_auc\")\n        else:\n            score=-1\n      \n        if (score > best_score[fold_num].min()):\n          \n            idx_min=np.argmin(best_score[fold_num])\n\n            best_score[fold_num][idx_min]=score\n            best_epoch[fold_num][idx_min]=epoch+1\n\n            path_best_model=f'best_model_fold_{self.fn}_{idx_min}.hdf5'\n            self.model.save(SAVE_FOLDER/path_best_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def setup_input(image, label, data, label_name):\n    \n    tab_data=[tf.cast(data[tfeat], dtype=tf.float32) for tfeat in tab_feats]\n    \n    tabular=tf.stack(tab_data)\n    \n    return {'inp1': image, 'inp2':  tabular}, label\ndef data_augment(data, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement \n    # in the next function (below), this happens essentially for free on TPU. \n    # Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    data['inp1'] = tf.image.random_flip_left_right(data['inp1'])\n    data['inp1'] = tf.image.random_flip_up_down(data['inp1'])\n    #image = tf.image.random_saturation(image, 0, 2)\n    \n    return data, label\ndef get_training_dataset(dataset):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    # the training dataset must repeat for several epochs\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    #dataset = dataset.repeat()\n    dataset = dataset.batch(BATCH_SIZE)\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTO)\n    \n    return dataset\ndef get_validation_dataset(dataset):\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    # prefetch next batch while training (autotune prefetch buffer size)\n    dataset = dataset.prefetch(AUTO)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndebug=1\nhistories = []\n\nbest_epoch={fn: np.zeros(NBEST) for fn in range(1, NFOLDS+1)}\nbest_score={fn: np.zeros(NBEST) for fn in range(1, NFOLDS+1)}\n\nfor fold_num in range(1, NFOLDS+1):\n    \n    tf.keras.backend.clear_session()\n    # clear tpu memory (otherwise can run into Resource Exhausted Error)\n    # see https://www.kaggle.com/c/flower-classification-with-tpus/discussion/131045\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    \n    print(\"=\"*50)\n    print(f\"Starting fold {fold_num} out of {NFOLDS}...\")\n    \n    files_trn=TRAIN_FNAMES[f\"fold_{fold_num}\"]\n    files_val=VAL_FNAMES[f\"fold_{fold_num}\"]\n    \n    if debug:\n        files_trn=files_trn[0:2]\n        files_val=files_val[0:2]\n        EPOCHS=1\n       \n    train_dataset = load_dataset(files_trn)\n    train_dataset = train_dataset.map(setup_input, num_parallel_calls=AUTO)\n    \n    val_dataset = load_dataset(files_val, ordered = True)\n    val_dataset = val_dataset.map(setup_input, num_parallel_calls=AUTO)\n    \n    model = get_model()\n    \n    STEPS_PER_EPOCH = count_data_items(files_trn) // BATCH_SIZE\n    \n    print(f'STEPS_PER_EPOCH = {STEPS_PER_EPOCH}')\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n    history = model.fit(get_training_dataset(train_dataset), \n                        steps_per_epoch=STEPS_PER_EPOCH, \n                        epochs=EPOCHS, \n                        callbacks=[lr_callback,\n                                   save_best_n(fold_num, model),\n                                   ],\n                        validation_data=get_validation_dataset(val_dataset),\n                        class_weight=class_weights,\n                        verbose=2,\n                       )\n    \n    idx_sorted=np.argsort(best_score[fold_num])\n    best_score[fold_num]=np.array(best_score[fold_num])[idx_sorted]\n    best_epoch[fold_num]=np.array(best_epoch[fold_num])[idx_sorted]\n\n    \n    \n    print(f\"\\nFold {fold_num} is finished. The best epochs: {[int(best_epoch[fold_num][i]) for i in range(len(best_epoch[fold_num]))]}\")\n    print(f\"The corresponding scores: {[round(best_score[fold_num][i], 5) for i in range(len(best_epoch[fold_num]))]}\")\n\n    histories.append(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_training_curves(fold_num, data):\n\n    plt.figure(figsize=(10,5), facecolor='#F0F0F0')\n\n    epochs=np.arange(1, EPOCHS+1)\n\n    # AUC\n    plt.plot(epochs, data['auc'], label='training auc', color='red')\n    plt.plot(epochs, data['val_auc'], label='validation auc', color='orange')\n\n    # Loss\n    plt.plot(epochs, data['loss'], label='training loss', color='blue')    \n    plt.plot(epochs, data['val_loss'], label='validation loss', color='green')\n\n    # Best\n    ls=['dotted', 'dashed', 'dashdot', 'solid'] # don't use more than 4 best epochs \n                                                # or make proper adjustments!\n    for i in range(NBEST):\n        plt.axvline(best_epoch[fold_num][i], 0, \n                    best_score[fold_num][i], linestyle=ls[i], \n                    color='black', label=f'AUC {best_score[fold_num][i]:.5f}')\n    \n    plt.title(f\"Fold {fold_num}. The best epochs: {[int(best_epoch[fold_num][i]) for i in range(len(best_epoch[fold_num]))]}; the best AUC's: {[round(best_score[fold_num][i], 5) for i in range(len(best_epoch[fold_num]))]}.\", \n              fontsize='14')\n    plt.ylabel('Loss/AUC', fontsize='12')\n    plt.xlabel('Epoch', fontsize='12')\n    plt.ylim((0, 1))\n    plt.legend(loc='lower left')\n    plt.tight_layout()\n    plt.show()\nfor fn in range(1, NFOLDS+1):\n    display_training_curves(fn, data=histories[fn-1].history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def setup_test_image(image, data):    \n    tab_data=[tf.cast(data[tfeat], dtype=tf.float32) for tfeat in tab_feats]\n    tabular=tf.stack(tab_data)\n\n    return {'inp1': image, 'inp2': tabular}\ndef setup_test_name(image, data):\n    return data['image_name']\ndef get_test_dataset(dataset):\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    \n    return dataset\ndef average_predictions(X, fn):\n    \n    y_probas=[]\n    \n    for idx in range(NBEST):\n        \n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        gc.collect()\n\n        print(f\"Predicting: fold {fn}, model {idx+1} out of {NBEST}...\")\n\n        with strategy.scope():\n            path_best_model=f'best_model_fold_{fn}_{idx}.hdf5'\n            model=tf.keras.models.load_model(SAVE_FOLDER/path_best_model)\n\n        y=model.predict(X)\n        y = rankdata(y)/len(y)\n        y_probas.append(y)\n    \n    y_probas=np.average(y_probas, axis=0)\n\n    return y_probas\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\npreds = pd.DataFrame({'image_name': np.zeros(len(test)), 'target': np.zeros(len(test))})\n\ntest_ds = load_dataset(TEST_FNAMES, labeled=False, ordered=True)\ntest_images_ds = test_ds.map(setup_test_image, num_parallel_calls=AUTO)\n\ntest_images_ds = get_test_dataset(test_images_ds)\ntest_ds = get_test_dataset(test_ds)\n\ntest_ids_ds = test_ds.map(setup_test_name, num_parallel_calls=AUTO).unbatch()\n\npreds['image_name'] = next(iter(test_ids_ds.batch(N_TEST_IMGS))).numpy().astype('U')\npreds['target'] = np.average([average_predictions(test_images_ds, fn) for fn in range(1, NFOLDS+1)], axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()\ndel sub['target']\nsub = sub.merge(preds, on='image_name')\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"The lengths of the submission file and `test` are {len(sub)} and {len(test)}, respectively.\")\nprint(f\"The number of NA's in the submission file is {sub.isna().sum().sum()}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if colab:\n    OUT_FOLDER=SAVE_FOLDER\nelse:\n    OUT_FOLDER=Path('')\n    \nsub.to_csv(OUT_FOLDER/'submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def setup_val_image(image, label, data, label_name):\n    \n    tab_data=[tf.cast(data[tfeat], dtype=tf.float32) for tfeat in tab_feats]\n    \n    tabular=tf.stack(tab_data)\n\n    return {'inp1': image, 'inp2': tabular}\ndef setup_val_name(image, label, data, label_name):\n    return data['image_name']\ndef setup_val_label(image, label, data, label_name):\n    return label\noof= pd.DataFrame({'image_name': train['image_name'].values})\n\nauc=[]\noof_all_folds=[]\n\nfor fold_num in range(1, NFOLDS+1):\n    \n    tf.keras.backend.clear_session()\n    \n    print(\"=\"*50)\n    print(f\"Starting fold {fold_num}...\")    \n\n    print(f\"The best epochs: {[int(best_epoch[fold_num][i]) for i in range(len(best_epoch[fold_num]))]}\")\n    print(f\"The corresponding scores: {[round(best_score[fold_num][i], 5) for i in range(len(best_epoch[fold_num]))]}\")\n\n    files_val=VAL_FNAMES[f\"fold_{fold_num}\"]\n    \n    if debug:\n        files_val=files_val[0:2]\n    \n    val_ds = load_dataset(files_val, ordered = True)\n    val_images_ds = val_ds.map(setup_val_image,\n                               num_parallel_calls=AUTO)\n    val_images_ds = get_validation_dataset(val_images_ds)\n        \n    val_ds = get_validation_dataset(val_ds)\n\n    val_label_ds = val_ds.map(setup_val_label,\n                              num_parallel_calls=AUTO).unbatch()\n    val_ids_ds = val_ds.map(setup_val_name,\n                            num_parallel_calls=AUTO).unbatch()\n    \n    n_val_fold = count_data_items(files_val)\n    \n    print(f'The # of validation files = {n_val_fold}')    \n    \n    oof_fold= pd.DataFrame()\n    \n    oof_fold['image_name'] = next(iter(val_ids_ds.batch(n_val_fold))).numpy().astype('U')\n\n    oof_fold['target'] = average_predictions(val_images_ds, fold_num)\n\n    oof_all_folds.append(oof_fold)\n\n    y_true = next(iter(val_label_ds.batch(n_val_fold))).numpy()\n\n    auc_fold=roc_auc_score(y_true, oof_fold['target'].values)\n    auc.append(auc_fold)\n    \n    print(f\"Fold {fold_num} is done! ROC AUC = {auc_fold:.5f}\")\n\noof=oof.merge(pd.concat(oof_all_folds), \n              on='image_name', \n              how='left').reset_index(drop=True)\n\nif debug:\n    oof=oof.fillna(0)\n\nauc=np.array(auc)\nauc_av=auc.mean()\nauc_std=auc.std()\n\nprint(f\"ROC AUC = {auc_av:.5f}, STDEV = {auc_std:.5f} (average across the folds)\")\n\nauc_oof=roc_auc_score(train['target'].values, oof['target'].values)\n\nprint(f\"ROC AUC = {auc_oof:.5f} (out of folds)\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"The lengths of the oof data frame and `train` are {len(oof)} and {len(train)}, respectively.\")\nprint(f\"The number of NA's in the oof data frame is {oof.isna().sum().sum()}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof.to_csv(OUT_FOLDER/'oof_1.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}