{"cells":[{"metadata":{},"cell_type":"markdown","source":"#  Stratified GroupKFold TFRecords \n\nAll the Dataset can be found at : [ [(128x128)](https://www.kaggle.com/prateek0x/ranzcr-128x128) , ([256x256](https://www.kaggle.com/prateek0x/ranzcr-256x256)) , ([384x384](https://www.kaggle.com/prateek0x/ranzcr-384x384/)) , ([512x512](https://www.kaggle.com/prateek0x/ranzcr-512x512)) ]\n\n**A sample notebook** that presents Stratified GroupKFold cross-validation and Efficient Net architecture getting trained using TPUs can be found [here.](https://www.kaggle.com/prateek0x/stratified-groupkfold-with-efn-tfrecords)\n\nFor discussions see this [thread](https://www.kaggle.com/c/ranzcr-clip-catheter-line-classification/discussion/208689)"},{"metadata":{},"cell_type":"markdown","source":"### Thanks to :\n1. Notebook :  [How to Create TFrecords](https://www.kaggle.com/dimitreoliveira/cassava-leaf-disease-stratified-tfrecords-256x256)\n2. Discussion : [A simple way to split folds](https://www.kaggle.com/c/ranzcr-clip-catheter-line-classification/discussion/204638)\n"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2020-12-05T18:11:25.720088Z","iopub.status.busy":"2020-12-05T18:11:25.719222Z","iopub.status.idle":"2020-12-05T18:11:33.311182Z","shell.execute_reply":"2020-12-05T18:11:33.310436Z"},"papermill":{"duration":7.619603,"end_time":"2020-12-05T18:11:33.311321","exception":false,"start_time":"2020-12-05T18:11:25.691718","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter, defaultdict\nfrom sklearn.model_selection import GroupKFold\nimport re, os, cv2, random, warnings, shutil,tqdm\n\ndef seed_everything(seed=0):\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.019644,"end_time":"2020-12-05T18:41:33.600516","exception":false,"start_time":"2020-12-05T18:41:33.580872","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Configuration"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-12-05T18:41:33.646995Z","iopub.status.busy":"2020-12-05T18:41:33.646268Z","iopub.status.idle":"2020-12-05T18:41:33.75213Z","shell.execute_reply":"2020-12-05T18:41:33.751022Z"},"papermill":{"duration":0.13174,"end_time":"2020-12-05T18:41:33.752309","exception":false,"start_time":"2020-12-05T18:41:33.620569","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"database_base_path = '../input/ranzcr-clip-catheter-line-classification/'\nPATH = f'{database_base_path}train/'\nIMGS = os.listdir(PATH)\nN_FILES = 15 # split images into 15 files\nHEIGHT, WIDTH = (256, 256) # Resized Image size\nIMG_QUALITY = 100 \n\nprint(f'Image samples: {len(IMGS)}')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.019744,"end_time":"2020-12-05T18:41:33.489169","exception":false,"start_time":"2020-12-05T18:41:33.469425","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Auxiliary functions"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:41:33.557508Z","iopub.status.busy":"2020-12-05T18:41:33.548741Z","iopub.status.idle":"2020-12-05T18:41:33.560646Z","shell.execute_reply":"2020-12-05T18:41:33.559845Z"},"papermill":{"duration":0.051443,"end_time":"2020-12-05T18:41:33.560772","exception":false,"start_time":"2020-12-05T18:41:33.509329","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n                      \n    image = tf.image.resize(image, [HEIGHT, WIDTH])\n    image = tf.reshape(image, [HEIGHT, WIDTH, 3])\n    return image\n\ndef read_tfrecord(example):\n    \n    TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"ETT - Abnormal\": tf.io.FixedLenFeature([], tf.int64),\n        \"ETT - Borderline\": tf.io.FixedLenFeature([], tf.int64),\n        \"ETT - Normal\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Abnormal\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Borderline\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Incompletely Imaged\": tf.io.FixedLenFeature([], tf.int64),\n        \"NGT - Normal\": tf.io.FixedLenFeature([], tf.int64),\n        \"CVC - Abnormal\": tf.io.FixedLenFeature([], tf.int64),\n        \"CVC - Borderline\": tf.io.FixedLenFeature([], tf.int64),\n        \"CVC - Normal\": tf.io.FixedLenFeature([], tf.int64),\n        \"Swan Ganz Catheter Present\": tf.io.FixedLenFeature([], tf.int64),\n        \"StudyInstanceUID\":tf.io.FixedLenFeature([], tf.string),\n        \"PatientID\":tf.io.FixedLenFeature([], tf.string)\n    \n    }\n    \n    example = tf.io.parse_single_example(example, TFREC_FORMAT)\n    image = decode_image(example['image'])\n    target = [example['ETT - Abnormal'],\n                 example['ETT - Borderline'],\n                 example['ETT - Normal'],\n                 example['NGT - Abnormal'],\n                 example['NGT - Borderline'],\n                 example['NGT - Incompletely Imaged'],\n                 example['NGT - Normal'],\n                 example['CVC - Abnormal'],\n                 example['CVC - Borderline'],\n                 example['CVC - Normal'],\n                 example['Swan Ganz Catheter Present']]\n    \n    \n    name = example['PatientID']\n    return image, target, name\n\ndef load_dataset(filenames, HEIGHT, WIDTH, CHANNELS=3):\n    dataset = tf.data.TFRecordDataset(filenames)\n    dataset = dataset.map(read_tfrecord, num_parallel_calls=AUTO)\n    return dataset\n\ndef display_samples(ds, row, col):\n    ds_iter = iter(ds)\n    plt.figure(figsize=(15, int(15*row/col)))\n    for j in range(row*col):\n        image, label, name = next(ds_iter)\n        plt.subplot(row,col,j+1)\n        plt.axis('off')\n        plt.imshow(image[0])\n        plt.title(name.numpy()[0], fontsize=12)\n    plt.show()\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n\n# Create TF Records\ndef _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image,ETT_Abnormal,ETT_Borderline,ETT_Normal, NGT_Abnormal, NGT_Borderline, NGT_Incompletely_Imaged, NGT_Normal, CVC_Abnormal,CVC_Borderline, CVC_Normal, SwanGanzCatheterPresent, unique_id, patient_id):\n    feature = {\n      'image': _bytes_feature(image),\n      'ETT - Abnormal':_int64_feature(ETT_Abnormal) ,\n      'ETT - Borderline':_int64_feature(ETT_Borderline),\n      'ETT - Normal':_int64_feature(ETT_Normal),\n      'NGT - Abnormal':_int64_feature(NGT_Abnormal),\n      'NGT - Borderline':_int64_feature(NGT_Borderline),\n      'NGT - Incompletely Imaged':_int64_feature(NGT_Incompletely_Imaged),\n      'NGT - Normal':_int64_feature(NGT_Normal),\n      'CVC - Abnormal':_int64_feature(CVC_Abnormal),\n      'CVC - Borderline':_int64_feature(CVC_Borderline),\n      'CVC - Normal':_int64_feature(CVC_Normal),\n      'Swan Ganz Catheter Present':_int64_feature(SwanGanzCatheterPresent),\n      'StudyInstanceUID': _bytes_feature(unique_id),\n      'PatientID': _bytes_feature(patient_id)\n    }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.020027,"end_time":"2020-12-05T18:41:33.793181","exception":false,"start_time":"2020-12-05T18:41:33.773154","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Load data"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:41:33.847122Z","iopub.status.busy":"2020-12-05T18:41:33.846381Z","iopub.status.idle":"2020-12-05T18:41:33.917544Z","shell.execute_reply":"2020-12-05T18:41:33.916775Z"},"papermill":{"duration":0.103902,"end_time":"2020-12-05T18:41:33.917666","exception":false,"start_time":"2020-12-05T18:41:33.813764","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv(database_base_path + 'train.csv')\nprint('Train samples: ', len(train))\n\ndisplay(train.head())","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.021549,"end_time":"2020-12-05T18:41:33.961276","exception":false,"start_time":"2020-12-05T18:41:33.939727","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Split samples into 15 different files"},{"metadata":{"trusted":true},"cell_type":"code","source":"target_cols = ['ETT - Abnormal', 'ETT - Borderline',\n       'ETT - Normal', 'NGT - Abnormal', 'NGT - Borderline',\n       'NGT - Incompletely Imaged', 'NGT - Normal', 'CVC - Abnormal',\n       'CVC - Borderline', 'CVC - Normal', 'Swan Ganz Catheter Present']","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:41:34.017817Z","iopub.status.busy":"2020-12-05T18:41:34.017045Z","iopub.status.idle":"2020-12-05T18:41:34.508532Z","shell.execute_reply":"2020-12-05T18:41:34.507714Z"},"papermill":{"duration":0.525356,"end_time":"2020-12-05T18:41:34.508661","exception":false,"start_time":"2020-12-05T18:41:33.983305","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"groups = np.array(train['PatientID'].values)\ngkf = GroupKFold(n_splits=N_FILES)\ntrain['fold'] = -1\n\nfor fold_n, (train_idx, val_idx) in enumerate(gkf.split(train[\"StudyInstanceUID\"],train[target_cols],groups)):\n    print('Fold: %s has %s samples' % (fold_n+1, len(val_idx)))\n    train['fold'].loc[val_idx] = fold_n\n    \ndisplay(train.head())\ntrain.to_csv('train.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.02331,"end_time":"2020-12-05T18:41:34.556106","exception":false,"start_time":"2020-12-05T18:41:34.532796","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Generate TF records"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:41:34.615797Z","iopub.status.busy":"2020-12-05T18:41:34.615052Z","iopub.status.idle":"2020-12-05T18:46:37.070472Z","shell.execute_reply":"2020-12-05T18:46:37.069767Z"},"papermill":{"duration":302.490823,"end_time":"2020-12-05T18:46:37.070634","exception":false,"start_time":"2020-12-05T18:41:34.579811","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"for tfrec_num in range(N_FILES):\n    print('\\nWriting TFRecord %i of %i...'%(tfrec_num, N_FILES))\n    samples = train[train['fold'] == tfrec_num]\n    n_samples = len(samples)\n    print(f'{n_samples} samples')\n    with tf.io.TFRecordWriter('Id_train%.2i-%i.tfrec'%(tfrec_num, n_samples)) as writer:\n        for row in tqdm.tqdm(samples.itertuples()):\n            \n            ETT_Abnormal = row._2\n            ETT_Borderline = row._3\n            ETT_Normal = row._4\n            NGT_Abnormal = row._5\n            NGT_Borderline = row._6\n            NGT_Incompletely_Imaged = row._7\n            NGT_Normal = row._8\n            CVC_Abnormal = row._9\n            CVC_Borderline = row._10\n            CVC_Normal = row._11\n            SwanGanzCatheterPresent = row._12\n            patient_id = row.PatientID\n            image_name = row.StudyInstanceUID+\".jpg\"\n            img_path = f'{PATH}{image_name}'\n            \n            img = cv2.imread(img_path)\n            img = cv2.resize(img, (HEIGHT, WIDTH))\n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, IMG_QUALITY))[1].tostring()\n            \n            example = serialize_example(img,ETT_Abnormal,ETT_Borderline,ETT_Normal, NGT_Abnormal, NGT_Borderline, NGT_Incompletely_Imaged, NGT_Normal, CVC_Abnormal,CVC_Borderline, CVC_Normal, SwanGanzCatheterPresent, str.encode(image_name), str.encode(patient_id))\n            \n            writer.write(example)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.029813,"end_time":"2020-12-05T18:46:37.130801","exception":false,"start_time":"2020-12-05T18:46:37.100988","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Visualize created TF records\n"},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-12-05T18:46:37.198465Z","iopub.status.busy":"2020-12-05T18:46:37.19773Z","iopub.status.idle":"2020-12-05T18:46:39.942076Z","shell.execute_reply":"2020-12-05T18:46:39.94268Z"},"papermill":{"duration":2.781566,"end_time":"2020-12-05T18:46:39.942843","exception":false,"start_time":"2020-12-05T18:46:37.161277","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nFILENAMES = tf.io.gfile.glob('Id_train*.tfrec')\nprint(f'TFRecords files: {FILENAMES}')\nprint(f'Created image samples: {count_data_items(FILENAMES)}')\n\ndisplay_samples(load_dataset(FILENAMES, HEIGHT, WIDTH).batch(1), 6, 6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Samples in each TFRecord"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(16, 6))\n\nax = sns.countplot(train['fold'])\n\nax.tick_params(axis='x', labelsize=20)\nax.tick_params(axis='y', labelsize=20)\nax.set_xticklabels([f'{value} ' for value in range(1,16)])\nax.set_xlabel('Folds', size=20, labelpad=20)\nax.set_ylabel('Samples', size=20, labelpad=20)\n\nplt.title(f'Training Set Number of Samples in Folds', size=20, pad=20)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.067973,"end_time":"2020-12-05T18:46:40.568192","exception":false,"start_time":"2020-12-05T18:46:40.500219","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## Labels distribution for each TFRecord"},{"metadata":{"trusted":true},"cell_type":"code","source":"tfrec_1 = train[train[\"fold\"]==0]\nlabel_dist = dict()\nfor tar in target_cols:\n    label_dist[tar] = int(tfrec_1[tar].sum())\n    \nfig = plt.figure(figsize=(7, 8), dpi=100)\n\nax = sns.barplot(y=[\"Count : \"+str(list(label_dist.values())[value])+\" : \"+list(label_dist.keys())[value] for value in range(0,11)],\n            x=list(label_dist.values()),\n            hue=list(label_dist.keys()),\n           )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"splits = train.groupby('fold').sum()[target_cols] \\\n        .reset_index(drop=True) \\\n        .T \\\n        .rename(columns={fold - 1: fold for fold in sorted(train['fold'].unique())}) \\\n        .reset_index() \\\n        .rename(columns={'index': 'Target'})\n\nsplits = pd.melt(splits, id_vars=['Target'], value_name='Count')\nsplits['Total'] = splits.groupby('Target')['Count'].transform('sum')\nsplits = splits.sort_values(by=['Total', 'Target'], ascending=False).reset_index(drop=True)\nsplits['variable'] = 'Fold ' + splits['variable'].astype(str)\n\nfig = plt.figure(figsize=(16, 20), dpi=100)\n\nsns.barplot(x=splits['Count'],\n            y=splits['Target'],\n            hue=splits['variable'])\n\nplt.xlabel('')\nplt.ylabel('')\nplt.tick_params(axis='x', labelsize=15)\nplt.tick_params(axis='y', labelsize=15)\nplt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0, prop={'size': 20})\nplt.title('Multi Label Stratified GroupKFold Target Counts', size=18, pad=18)\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}