{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [SIIM-FISABIO-RSNA COVID-19 Detection](https://www.kaggle.com/c/siim-covid19-detection)\n> Identify and localize COVID-19 abnormalities on chest radiographs\n\n![](https://storage.googleapis.com/kaggle-competitions/kaggle/26680/logos/header.png)","metadata":{}},{"cell_type":"markdown","source":"# Reference\nCheck this amazing notebook, [How To Create TFRecords](https://www.kaggle.com/cdeotte/how-to-create-tfrecords) by [Chris Deotte](https://www.kaggle.com/cdeotte)","metadata":{}},{"cell_type":"markdown","source":"# How to Create TFRecord","metadata":{}},{"cell_type":"code","source":"SEED  = 42\nFOLDS = 5\nDIM   = 512","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing Packages","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os, shutil\nfrom glob import glob\nfrom sklearn.cluster import KMeans\nfrom tqdm.notebook import tqdm\nfrom sklearn.preprocessing import LabelEncoder\nimport random\ntqdm.pandas()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(SEED)\nrandom.seed(SEED)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f'../input/siimcovid19-{DIM}-jpg-image-dataset/train.csv')\ntrain_df['image_path'] = F'../input/siimcovid19-{DIM}-jpg-image-dataset/train/'+train_df.image_id+'.jpg'\ntrain_df.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ClassName and ClassLabel Map","metadata":{}},{"cell_type":"code","source":"name2label = {'Typical Appearance': 3,\n 'Indeterminate Appearance': 1,\n 'Atypical Appearance': 2,\n 'Negative for Pneumonia': 0}\nclass_names = list(name2label.keys())\nlabel2name = {v:k for k, v in name2label.items()}\ntrain_df['class_name']  = train_df.progress_apply(lambda row:row[class_names].iloc[[row[class_names].values.argmax()]].index.tolist()[0], axis=1)\ntrain_df['class_label'] = train_df.class_name.map(name2label)\ntrain_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Stratified KFold by Groups","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold, StratifiedKFold\ngkf  = GroupKFold(n_splits = 5)\ntrain_df['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(train_df, groups = train_df.StudyInstanceUID.tolist())):\n    train_df.loc[val_idx, 'fold'] = fold\ntrain_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<!-- # Check the signals\nFrom the [dataset information](https://www.kaggle.com/c/seti-breakthrough-listen/overview/data-information),\n>>\nNot all of the “needle” signals look like diagonal lines, and they may not be present for the entirety of all three “A” observations, but what they do have in common is that they are only present in some or all of the “A” observations (panels **1**, **3**, and **5** in the cadence snippets).\n\n -->","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport cv2\ndef load_image(path, dim=DIM, ch=3):\n    img = cv2.imread(path, cv2.IMREAD_GRAYSCALE if ch==None else cv2.IMREAD_COLOR)\n    if img.shape[:2]!=(dim,dim) and dim!=-1:\n        img = cv2.resize(img, dsize=(dim,dim), interpolation=cv2.INTER_AREA)\n    return img\n\nplt.imshow(load_image(train_df.image_path.iloc[100], dim=-1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Channels","metadata":{}},{"cell_type":"markdown","source":"# TFRecord Data","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Writng TFRecord (Train)","metadata":{}},{"cell_type":"code","source":"def train_serialize_example(feature0, feature1, feature2, feature3):\n    feature = {\n      'image'         : _bytes_feature(feature0),\n      'image_id'      : _bytes_feature(feature1),\n      'group'         : _bytes_feature(feature2),    \n      'target'        : _int64_feature(feature3),\n  }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show=True\nfolds = train_df.fold.unique().tolist()\nfor fold in tqdm(folds): # create tfrecord for each fold\n    fold_df = train_df[train_df.fold==fold]\n    if show:\n        print(); print('Writing TFRecord of fold %i :'%(fold))  \n    with tf.io.TFRecordWriter('train%.2i-%i.tfrec'%(fold,fold_df.shape[0])) as writer:\n        samples = fold_df.shape[0]\n#         samples = 200\n        it = tqdm(range(samples)) if show else range(samples)\n        for k in it: # images in fold\n            row = fold_df.iloc[k,:]\n            image      = load_image(row['image_path'], dim=DIM)\n            image_id   = row['image_id']\n            group      = row['StudyInstanceUID']\n            target     = np.array(row['class_label'], dtype=np.uint8)\n            example  = train_serialize_example(\n                cv2.imencode('.jpg', image, (cv2.IMWRITE_JPEG_QUALITY, 96))[1].tobytes(),\n                str.encode(image_id),\n                str.encode(group),\n                target,\n                )\n            writer.write(example)\n        if show:\n            filepath = 'train%.2i-%i.tfrec'%(fold,fold_df.shape[0])\n            filename = filepath.split('/')[-1]\n            filesize = os.path.getsize(filepath)/10**6\n            print(filename,':',np.around(filesize, 2),'MB')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Writing TFRecord (Test)","metadata":{}},{"cell_type":"code","source":"# def test_serialize_example(feature0, feature1, feature2):\n#     feature = {\n#       'image'         : _bytes_feature(feature0),\n#       'image_id'      : _bytes_feature(feature1),\n#       'group'         : _bytes_feature(feature2),    \n#   }\n#     example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n#     return example_proto.SerializeToString()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show  = True\n# folds = 10\n# l     = int(np.ceil(test_df.shape[0]/folds))\n# for fold in tqdm(range(folds)): # create tfrecord for each fold\n#     fold_df = test_df.iloc[l*fold:l*(fold+1)]\n#     if show:\n#         print(); print('Writing TFRecord of fold %i :'%(fold))  \n#     with tf.io.TFRecordWriter('test%.2i-%i.tfrec'%(fold,fold_df.shape[0])) as writer:\n#         samples = fold_df.shape[0]\n# #         samples = 200\n#         it = tqdm(range(samples)) if show else range(samples)\n#         for k in it: # images in fold\n#             row = fold_df.iloc[k,:]\n#             image      = load_signal(row['filepath'], dim=DIM)\n#             image      = image[...,::-1] # rgb -> bgr, we'll get the rgb form after decoding the tfrec\n#             image_id   = row['id']\n#             group      = row['group']\n#             example  = test_serialize_example(\n#                 cv2.imencode('.png', image)[1].tobytes(),\n#                 str.encode(image_id),\n#                 str.encode(group),\n#                 )\n#             writer.write(example)\n#         if show:\n#             filepath = 'test%.2i-%i.tfrec'%(fold,fold_df.shape[0])\n#             filename = filepath.split('/')[-1]\n#             filesize = os.path.getsize(filepath)/10**6\n#             print(filename,':',np.around(filesize, 2),'MB')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading TFRecord","metadata":{}},{"cell_type":"code","source":"import re, math\ndef decode_image(image_data):\n    image = tf.image.decode_png(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\ndef prepare_target(target):    \n    target = tf.cast(target, tf.float32)            \n    target = tf.reshape(target, [1])         \n    return target\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\" : tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    image  = tf.reshape(image, [DIM, DIM, 3])\n    target = prepare_target(example['target'])\n    return image, target # returns a dataset of (image, label) pairs\n\ndef load_dataset(fileids, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(fileids, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(20, seed=SEED)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(fileids):\n    # the number of data items is written in the id of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(fileid).group(1)) for fileid in fileids]\n    return np.sum(n)","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visual","metadata":{}},{"cell_type":"code","source":"def display_batch(batch, size=2):\n    imgs, tars = batch\n    plt.figure(figsize=(size*5, 5))\n    for img_idx in range(size):\n        plt.subplot(1, size, img_idx+1)\n        plt.title(f'class: {label2name[tars[img_idx].numpy()[0]]}', fontsize=15)\n        plt.imshow(imgs[img_idx,:, :, :])\n        plt.xticks([])\n        plt.yticks([])\n    plt.tight_layout()\n    plt.show() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Total Images","metadata":{}},{"cell_type":"code","source":"# INITIALIZE VARIABLES\nIMAGE_SIZE= [DIM,DIM];\nBATCH_SIZE = 32\nAUTO = tf.data.experimental.AUTOTUNE\nTRAINING_FILENAMES = tf.io.gfile.glob('train*.tfrec')\nTEST_FILENAMES     = tf.io.gfile.glob('test*.tfrec')\nprint('There are %i train & %i test images'%(count_data_items(TRAINING_FILENAMES), count_data_items(TEST_FILENAMES)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Once Batch Image","metadata":{}},{"cell_type":"code","source":"# DISPLAY TRAIN IMAGES\ntraining_dataset = get_training_dataset()\ntraining_dataset = training_dataset.unbatch().batch(20)\ntrain_batch = next(iter(training_dataset))\ndisplay_batch(train_batch, 5);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img, label = train_batch\nnp.unique(label.numpy(), return_counts=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}