{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install -q efficientnet\nimport os\nimport re\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom tensorflow.keras import backend as K\nimport tensorflow_addons as tfa\nfrom kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For tf.dataset\nAUTO = tf.data.experimental.AUTOTUNE\n\n# Data access\nGCS_PATH = KaggleDatasets().get_gcs_path('ranzcr-tf-records-512-stratified')\n\n# Configuration\nEPOCHS = 13\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [512, 512]\n# Seed\nSEED = 321\n# Learning rate\nLR = 0.001\n# Test time augmentation rounds\nTTA = 5\n# Verbosity\nVERBOSE = 2\n# Number of classes\nN_CLASSES = 11\n\n# Training filenames directory\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/*.tfrec')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n\n# Data augmentation function\ndef data_augment(image, StudyInstanceUID, targets):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_saturation(image, 0.70, 1.30)\n    image = tf.image.random_contrast(image, 0.80, 1.20)\n    image = tf.image.random_brightness(image, 0.10)\n    return image, StudyInstanceUID, targets\n\n# Function to decode our images\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.resize(image, IMAGE_SIZE)\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\n# This function parse our images and also get the target variable\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"StudyInstanceUID\": tf.io.FixedLenFeature([], tf.string),\n        \"ETT - Abnormal\": tf.io.FixedLenFeature([], tf.int64),\n        \"ETT - Borderline\": tf.io.FixedLenFeature([], tf.int64),\n        \"ETT - Normal\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Abnormal\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Borderline\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Incompletely Imaged\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Normal\": tf.io.FixedLenFeature([], tf.int64),\n        \"CVC - Abnormal\": tf.io.FixedLenFeature([], tf.int64),\n        \"CVC - Borderline\": tf.io.FixedLenFeature([], tf.int64),\n        \"CVC - Normal\": tf.io.FixedLenFeature([], tf.int64),\n        \"Swan Ganz Catheter Present\": tf.io.FixedLenFeature([], tf.int64)\n    }\n\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    StudyInstanceUID = example['StudyInstanceUID']\n    ETT_Abnormal = tf.cast(example['ETT - Abnormal'], tf.float32)\n    ETT_Borderline = tf.cast(example['ETT - Borderline'], tf.float32)\n    ETT_Normal = tf.cast(example['ETT - Normal'], tf.float32)\n    NGT_Abnormal = tf.cast(example['NGT - Abnormal'], tf.float32)\n    NGT_Borderline = tf.cast(example['NGT - Borderline'], tf.float32)\n    NGT_Incompletely_Imaged = tf.cast(example['NGT - Incompletely Imaged'], tf.float32)\n    NGT_Normal = tf.cast(example['NGT - Normal'], tf.float32)\n    CVC_Abnormal = tf.cast(example['CVC - Abnormal'], tf.float32)\n    CVC_Borderline = tf.cast(example['CVC - Borderline'], tf.float32)\n    CVC_Normal = tf.cast(example['CVC - Normal'], tf.float32)\n    Swan_Ganz_Catheter_Present = tf.cast(example['Swan Ganz Catheter Present'], tf.float32)\n    targets = tf.stack([ETT_Abnormal] + [ETT_Borderline] + [ETT_Normal] + [NGT_Abnormal] + [NGT_Borderline] + [NGT_Incompletely_Imaged] + \\\n                       [NGT_Normal] + [CVC_Abnormal] + [CVC_Borderline] + [CVC_Normal] + [Swan_Ganz_Catheter_Present])\n    return image, StudyInstanceUID, targets\n\n# This function loads TF Records and parse them into tensors\ndef load_dataset(filenames, ordered = False):\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False \n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls = AUTO) \n    return dataset\n\n# This function is to get our training tensors\ndef get_training_dataset(filenames, ordered = False):\n    dataset = load_dataset(filenames, ordered = ordered)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n# This function is to get our validation tensors\ndef get_validation_dataset(filenames, ordered = True):\n    dataset = load_dataset(filenames, ordered = ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) \n    return dataset\n\n# This function is to get our validation time augmentation tensors\ndef get_vta(filenames, ordered = True):\n    dataset = load_dataset(filenames, ordered = ordered)\n    dataset = dataset.map(data_augment, num_parallel_calls = AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\n# Function to count how many photos we have in\ndef count_data_items(filenames):\n    # The number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nprint(f'Dataset: {NUM_TRAINING_IMAGES} training images')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to get the mean roc auc from a multi label matrix\ndef mean_roc_auc(targets, probabilities):\n    roc_auc = []\n    for k in range(N_CLASSES):\n        roc_auc.append(metrics.roc_auc_score(targets[:, k], probabilities[:, k]))\n    return np.average(roc_auc)\n        \n# Function to create our EfficientnetB6 model\ndef get_model():\n    \n    with strategy.scope():\n        \n        inp = tf.keras.layers.Input(shape = (*IMAGE_SIZE, 3))\n\n        x = efn.EfficientNetB7(weights = 'imagenet', include_top = False)(inp)\n        x = tf.keras.layers.GlobalAveragePooling2D()(x)\n        output = tf.keras.layers.Dense(N_CLASSES, activation = 'sigmoid')(x)\n        \n        model = tf.keras.models.Model(inputs = [inp], outputs = [output])\n\n        opt = tf.keras.optimizers.Adam(learning_rate = LR)\n\n        model.compile(\n            optimizer = opt,\n            loss = [tf.keras.losses.BinaryCrossentropy()],\n            metrics = [tf.keras.metrics.AUC(multi_label = True)]\n        )\n\n        return model\n\n# Function to train and evaluate our model\ndef train_and_evaluate(folds = 5):\n    oof_StudyInstanceUID = []\n    oof_targets = np.zeros((NUM_TRAINING_IMAGES, N_CLASSES))\n    oof_predictions = np.zeros((NUM_TRAINING_IMAGES, N_CLASSES))\n    previous_number_of_files = 0\n    total_number_of_files = 0\n    \n    # Seed everything\n    seed_everything(SEED)\n    kfold = KFold(folds, shuffle = True, random_state = SEED)\n    for fold, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n        if tpu:\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n        print('\\n')\n        print('-'*50)\n        print(f'Training fold {fold + 1}')\n        train_dataset = get_training_dataset([TRAINING_FILENAMES[x] for x in trn_ind], ordered = False)\n        train_dataset = train_dataset.map(lambda image, StudyInstanceUID, targets: (image, targets))\n        val_dataset = get_validation_dataset([TRAINING_FILENAMES[x] for x in val_ind], ordered = True)\n        val_dataset_ = val_dataset.map(lambda image, StudyInstanceUID, targets: (image, targets))\n        STEPS_PER_EPOCH = count_data_items([TRAINING_FILENAMES[x] for x in trn_ind]) // BATCH_SIZE\n        K.clear_session()\n        model = get_model()\n        # Model checkpoint\n        checkpoint = tf.keras.callbacks.ModelCheckpoint(f'EfficientNetB7_{fold}_{IMAGE_SIZE[0]}_{SEED}.h5', \n                                                        monitor = 'val_loss', \n                                                        verbose = VERBOSE, \n                                                        save_best_only = True,\n                                                        save_weights_only = True, \n                                                        mode = 'min')\n        lr_scheduler = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_auc', \n                                                            factor = 0.1, \n                                                            patience = 2, \n                                                            verbose = VERBOSE,\n                                                            mode = 'max')\n        history = model.fit(train_dataset,\n                            steps_per_epoch = STEPS_PER_EPOCH,\n                            epochs = EPOCHS,\n                            callbacks = [checkpoint, lr_scheduler], \n                            validation_data = val_dataset_,\n                            verbose = VERBOSE)\n        \n        # Load weights from the best epoch\n        model.load_weights(f'EfficientNetB7_{fold}_{IMAGE_SIZE[0]}_{SEED}.h5')\n        \n        # Get ids and targets\n        number_of_files = count_data_items([TRAINING_FILENAMES[x] for x in val_ind])\n        StudyInstanceUID = val_dataset.map(lambda image, StudyInstanceUID, targets: StudyInstanceUID).unbatch()\n        targets = val_dataset.map(lambda image, StudyInstanceUID, targets: targets).unbatch()\n        StudyInstanceUID = next(iter(StudyInstanceUID.batch(number_of_files))).numpy().astype('U')\n        targets = next(iter(targets.batch(number_of_files))).numpy()\n        oof_StudyInstanceUID.extend(list(StudyInstanceUID))\n        total_number_of_files += number_of_files\n        oof_targets[previous_number_of_files:total_number_of_files] = targets\n        \n        # Use validation time augmentation for predictions\n        steps = TTA * number_of_files / BATCH_SIZE\n        dataset = get_vta([TRAINING_FILENAMES[x] for x in val_ind], ordered = True)\n        image = dataset.map(lambda image, StudyInstanceUID, targets: image)\n        probabilities = model.predict(image, steps = steps)[: TTA * number_of_files]\n        probabilities = np.mean(probabilities.reshape((number_of_files, TTA, N_CLASSES), order = 'F'), axis = 1)\n        oof_predictions[previous_number_of_files:total_number_of_files] = probabilities\n        previous_number_of_files += number_of_files\n        \n        print('\\n')\n        print('-'*50)\n        fold_roc_auc_score = mean_roc_auc(targets, probabilities)\n        print(f'Our fold {fold + 1} roc auc score validation with {TTA} TTA is {fold_roc_auc_score}')\n        \n    print('\\n')\n    print('-'*50)\n    oof_roc_auc_score = mean_roc_auc(oof_targets, oof_predictions)\n    print(f'Our out of folds roc auc score is {oof_roc_auc_score}')\n    \n    # Save the out of folds predictions\n    print('Saving out of folds to disk...')\n    target_columns = [\"ETT - Abnormal\", \"ETT - Borderline\", \"ETT - Normal\", \"NGT - Abnormal\", \"NGT - Borderline\", \"NGT - Incompletely Imaged\", \"NGT - Normal\", \"CVC - Abnormal\", \"CVC - Borderline\", \n                      \"CVC - Normal\", \"Swan Ganz Catheter Present\"]\n    prediction_columns = [col + ' Prob' for col in target_columns]\n    oof_targets_df = pd.DataFrame(oof_targets, columns = target_columns)\n    oof_predictions_df = pd.DataFrame(oof_predictions, columns = prediction_columns)\n    oof_dataset = pd.DataFrame({'oof_StudyInstanceUID': oof_StudyInstanceUID})\n    oof_dataset = pd.concat([oof_dataset, oof_targets_df, oof_predictions_df], axis = 1)\n    oof_dataset.to_csv(f'EfficientNetB7_{IMAGE_SIZE[0]}_{SEED}.csv', index = False)\n    \ntrain_and_evaluate(folds = 5)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}