{"cells":[{"metadata":{},"cell_type":"markdown","source":"* Ex. test_fold_3_0-1024.tfrec : \n\n    1. test tfrecord\n    2. 3rd fold\n    3. 0th file sub_id\n    4. 1024 tfrecord contains 1024 image","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### This Version contains Stratified-GroupKFold=5 **Image_size=(1024,1024,3)** Image-quality=90","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# <font color='Blue'>Feel free to fork kernel and use SGK-Fold TFRecords + Tabular Data (One-Hot encoded) + Different Shape Images</font>","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Import Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport random, re, math, gc\nfrom collections import Counter, defaultdict\n\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\n\nimport tensorflow as tf, tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\nAUTO = tf.data.experimental.AUTOTUNE\n\nprint(tf.__version__)\nprint(tf.keras.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Set Seed","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def seed_everything(seed=1234):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Set Meta-Data and Load-CSV-Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"image_shape = (1024,1024,3)\nimage_quality = 90\nSEED = 1234\nseed_everything(SEED)\nBATCH_SIZE = 16 \n\ntrain_fold = 5\ntest_fold = 6\n\n\nBASE_PATH = '../input/siim-isic-melanoma-classification'\ntrain_metadata = pd.read_csv(os.path.join(BASE_PATH, 'train.csv'))\ntest_metadata = pd.read_csv(os.path.join(BASE_PATH, 'test.csv'))\nsample_submission = pd.read_csv(os.path.join(BASE_PATH, 'sample_submission.csv'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Overview","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_metadata.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_metadata.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Missing Values in Train_metadata in(%): \\n')\nmissing_values = train_metadata.isnull().sum() / len(train_metadata)\nmissing_values = missing_values[missing_values>0.0]\nprint(missing_values)\n\nprint('\\n\\nMissing Values in Test_metadata : \\n')\nmissing_values = test_metadata.isnull().sum() / len(test_metadata)\nmissing_values = missing_values[missing_values>0.0]\nprint(missing_values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Unique values in column with frequency : ')\n\nprint('\\nsex : ', dict(train_metadata.sex.value_counts()))\nprint('\\nage_approx : ', dict(train_metadata.age_approx.value_counts()))\nprint('\\nanatom_site_general_challenge : ', dict(train_metadata.anatom_site_general_challenge.value_counts()))\nprint('\\ndiagnosis : ', dict(train_metadata.diagnosis.value_counts()))\nprint('\\nbenign_malignant : ', dict(train_metadata.benign_malignant.value_counts()))\nprint('\\ntarget : ', dict(train_metadata.target.value_counts()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get Stratify Group Index","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_stratify_group(row):\n    return '{}_{}_{}'.format(row['anatom_site_general_challenge'],row['sex'],row['target'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fill NAN Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train_metadata.copy()\n\ntrain_cat_list_patient_id = train['patient_id'].astype('category').cat.categories\ntrain['patient_id'] = train.patient_id.astype('category').cat.codes\n\ntrain['sex'] = train['sex'].fillna('unknown')\n# cat_list_sex = train['sex'].astype('category').cat.categories\n# train['sex'] = train.sex.astype('category').cat.codes\n\ntrain['age_approx'] = train['age_approx'].fillna(train.age_approx.mean())\ntrain['age_approx'] = train['age_approx'].astype('int')\n\ntrain['anatom_site_general_challenge'] = train['anatom_site_general_challenge'].fillna(\"unknown\")\n# cat_list_anatom_site_general_challenge = train['anatom_site_general_challenge'].astype('category').cat.categories\n# train['anatom_site_general_challenge'] = train['anatom_site_general_challenge'].astype('category').cat.codes\n\ntrain['stratify_group'] = train.fillna(\"NA\").apply(get_stratify_group, axis=1)\ntrain['stratify_group'] = train['stratify_group'].astype('category').cat.codes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test_metadata.copy()\n\ntest_cat_list_patient_id = test['patient_id'].astype('category').cat.categories\ntest['patient_id'] = test.patient_id.astype('category').cat.codes\n\ntest['anatom_site_general_challenge'] = test['anatom_site_general_challenge'].fillna(\"unknown\")\n# test_cat_list_anatom_site_general_challenge = test['anatom_site_general_challenge'].astype('category').cat.categories\n# test['anatom_site_general_challenge'] = test['anatom_site_general_challenge'].astype('category').cat.codes\n\ntest['target'] = 2\n\ntest['stratify_group'] = test.fillna(\"NA\").apply(get_stratify_group, axis=1)\ntest['stratify_group'] = test['stratify_group'].astype('category').cat.codes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stratified Group k-fold Function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def stratified_group_k_fold(X, y, groups, k, seed=None):\n    labels_num = np.max(y) + 1\n    y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))\n    y_distr = Counter()\n    for label, g in zip(y, groups):\n        y_counts_per_group[g][label] += 1\n        y_distr[label] += 1\n\n    y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))\n    groups_per_fold = defaultdict(set)\n\n    def eval_y_counts_per_fold(y_counts, fold):\n        y_counts_per_fold[fold] += y_counts\n        std_per_label = []\n        for label in range(labels_num):\n            label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])\n            std_per_label.append(label_std)\n        y_counts_per_fold[fold] -= y_counts\n        return np.mean(std_per_label)\n    \n    groups_and_y_counts = list(y_counts_per_group.items())\n    random.Random(seed).shuffle(groups_and_y_counts)\n\n    for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])):\n        best_fold = None\n        min_eval = None\n        for i in range(k):\n            fold_eval = eval_y_counts_per_fold(y_counts, i)\n            if min_eval is None or fold_eval < min_eval:\n                min_eval = fold_eval\n                best_fold = i\n        y_counts_per_fold[best_fold] += y_counts\n        groups_per_fold[best_fold].add(g)\n\n    all_groups = set(groups)\n    \n    for i in range(k):\n        train_groups = all_groups - groups_per_fold[i]\n        test_groups = groups_per_fold[i]\n\n        train_indices = [i for i, g in enumerate(groups) if g in train_groups]\n        test_indices = [i for i, g in enumerate(groups) if g in test_groups]\n\n        yield train_indices, test_indices","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training-Data Stratified Group k fold","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain['fold'] = 0\n\n\nfor fold_ind, (train_ind, val_ind) in enumerate(stratified_group_k_fold(train_metadata, \n                                                                        train.stratify_group.values, \n                                                                        train_metadata.patient_id.values, \n                                                                        k=train_fold, \n                                                                        seed=SEED)):\n    train.loc[val_ind,'fold'] = fold_ind\n\ntrain.fold.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training One-Hot-Encoding","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sex_code = pd.get_dummies(train.sex, prefix='sex')\nanatom_site_general_challenge_code = pd.get_dummies(train.anatom_site_general_challenge, prefix='anatom_site')\nage_aprox_normalized = (train.age_approx-train.age_approx.mean())/train.age_approx.std()\ntrain_coded = pd.concat([train.image_name, sex_code, age_aprox_normalized ,anatom_site_general_challenge_code, train.target, train.fold], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_coded.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Encoded-Train-Data Columns","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_coded.columns))\ntrain_coded.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training-Data Stratified Group k fold","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntest['fold'] = 0\n\nfor fold_ind, (test_ind, val_ind) in enumerate(stratified_group_k_fold(test_metadata, \n                                                                       test.stratify_group.values, \n                                                                       test_metadata.patient_id.values, \n                                                                       k=test_fold, \n                                                                       seed=SEED)):\n    test.loc[val_ind,'fold'] = fold_ind\n\ntest.fold.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Testing One-Hot-Encoding","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sex_code = pd.get_dummies(test.sex, prefix='sex')\nanatom_site_general_challenge_code = pd.get_dummies(test.anatom_site_general_challenge, prefix='anatom_site')\nage_aprox_normalized = (test.age_approx-test.age_approx.mean())/test.age_approx.std()\ntest_coded = pd.concat([test.image_name, sex_code, age_aprox_normalized ,anatom_site_general_challenge_code, test.fold], axis=1)\ntest_coded['sex_unknown'] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_coded.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Encoded-Test-Data Columns","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(test_coded.columns))\ntest_coded.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Write TFRecords**\n sorce from TensorFlow's docs [here](https://www.tensorflow.org/tutorials/load_data/tfrecord)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_path(image_name, _set='train'):\n    if _set == 'train':    \n        return BASE_PATH + '/jpeg/train/' + str(image_name) + '.jpg'\n    else:\n        return BASE_PATH + '/jpeg/test/' + str(image_name) + '.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def row_serialize(row, image_shape, _set='train'):\n    \n    image_path = get_path(row['image_name'],_set=_set)\n    \n    ## Method -1\n    \n#     raw_image = tf.keras.preprocessing.image.load_img(image_path,\n#                                                       color_mode='rgb',\n#                                                       target_size=image_shape\n#                                                      ).tobytes()\n\n    ## Method-2 (with 10% compression)\n    \n    image = tf.io.read_file(image_path)\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.image.resize(image, image_shape[:2], method=\"nearest\")\n    raw_image = tf.image.encode_jpeg(image, quality=image_quality, optimize_size=True)\n    del image\n    \n    ## Method-3 (with 5% compression)\n    \n#     image = cv2.imread(image_path)\n#     image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)\n#     image = cv2.resize(image,image_shape[:2])\n#     result, encimg = cv2.imencode('.jpg', image, [int(cv2.IMWRITE_JPEG_QUALITY), 90])\n#     raw_image = encimg.tostring()\n#     del image,result\n\n    feature = {\n        # _bytes_feature\n        'image': _bytes_feature(raw_image),\n        'image_name': _bytes_feature(str.encode(row['image_name'])),\n        \n        # _int64_feature\n        'sex_female': _int64_feature(row['sex_female']),\n        'sex_male': _int64_feature(row['sex_male']),\n        'sex_unknown': _int64_feature(row['sex_unknown']),\n        'anatom_site_head/neck': _int64_feature(row['anatom_site_head/neck']),\n        'anatom_site_lower extremity': _int64_feature(row['anatom_site_lower extremity']),\n        'anatom_site_oral/genital': _int64_feature(row['anatom_site_oral/genital']),\n        'anatom_site_palms/soles': _int64_feature(row['anatom_site_palms/soles']),\n        'anatom_site_torso': _int64_feature(row['anatom_site_torso']),\n        'anatom_site_unknown': _int64_feature(row['anatom_site_unknown']),\n        'anatom_site_upper extremity': _int64_feature(row['anatom_site_upper extremity']),\n        \n        # _float_feature\n        'age_approx': _float_feature(row['age_approx']),\n      }\n    \n    \n    if _set=='train':\n        feature['target'] = _int64_feature(row['target'])\n\n    \n    row_feature = tf.train.Example(features=tf.train.Features(feature=feature))\n    return row_feature.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make Folder train_fold_tfrecords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_fold_path = '../working/train_fold_tfrecords_{}x{}'.format(image_shape[0],image_shape[1])\ntry:\n    os.mkdir(train_fold_path)\nexcept OSError as error: \n    print(error, \"\\nIt's OK You are Good to GO\")     ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Write Trainging TFRecords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nMAX_size = 1024\nfor fold, group in train_coded.groupby('fold'):\n    Max_sub_files = len(group)//MAX_size\n    print('\\nWriting Fold : ',fold)\n    for file_id in range(Max_sub_files+1):\n        \n        if file_id != Max_sub_files:\n            num_image = MAX_size\n        else:\n            num_image = len(group)%MAX_size\n            \n        with tf.io.TFRecordWriter(train_fold_path + '/tain_fold_{}_{}-{}.tfrec'.format(fold,file_id,num_image)) as writer:\n#             print(file_id*MAX_size,file_id*MAX_size + num_image, num_image)\n            for i in range(file_id*MAX_size,file_id*MAX_size + num_image):\n                row_feature_serialize = row_serialize(group.iloc[i], image_shape, _set='train')\n                writer.write(row_feature_serialize)\n                del row_feature_serialize\n\n                if i%100 == 0:\n                    print(i,', ',end='')\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Time required for 1 fold(1935 images) [CPU]\n\n*  **Method-1**\n    1. CPU times: user 16min 34s, sys: 53.4 s, total: 17min 28s\n    2. Wall time: 17min 29s\n\n\n*  **Method-2**\n    1. CPU times: user 2min 14s, sys: 38 s, total: 2min 52s\n    2. Wall time: 2min 42s\n\n\n*  **Mehod-3**\n    1. CPU times: user 6min, sys: 1min 35s, total: 7min 35s\n    2. Wall time: 6min 44s","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Make Folder test_fold_tfrecords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_fold_path = '../working/test_fold_tfrecords_{}x{}'.format(image_shape[0],image_shape[1])\n# try:\n#     os.mkdir(test_fold_path)\n# except OSError as error: \n#     print(error, \"\\nIt's OK You are Good to GO\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Write Testing TFRecords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# MAX_size = 1024\n# for fold, group in test_coded.groupby('fold'):\n#     Max_sub_files = len(group)//MAX_size\n#     print('\\nWriting Fold : ',fold)\n#     for file_id in range(Max_sub_files+1):\n        \n#         if file_id != Max_sub_files:\n#             num_image = MAX_size\n#         else:\n#             num_image = len(group)%MAX_size\n            \n#         with tf.io.TFRecordWriter(test_fold_path + '/test_fold_{}_{}-{}.tfrec'.format(fold,file_id,num_image)) as writer:\n# #             print(file_id*MAX_size,file_id*MAX_size + num_image, num_image)\n#             for i in range(file_id*MAX_size,file_id*MAX_size + num_image):\n#                 row_feature_serialize = row_serialize(group.iloc[i], image_shape, _set='test')\n#                 writer.write(row_feature_serialize)\n#                 del row_feature_serialize\n\n#                 if i%100 == 0:\n#                     print(i,', ',end='')\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TEMP CODE (NEED TO REMOVE)\n\n# train_fold = 6\n# test_fold = 2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Read TFRecords","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Make train-validation fold","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# all_train_tfrecords_path = tf.io.gfile.glob(train_fold_path + '/*.tfrec')\n# test_tfrecords_path = tf.io.gfile.glob(test_fold_path + '/*.tfrec')\n\n# val_tfrecords_path = {}\n# train_tfrecords_path = {}\n\n# for i in range(train_fold):\n#     val_tfrecords_path['fold_{}'.format(i)] = [path for path in all_train_tfrecords_path if f\"tain_fold_{i}_\" in path] \n    \n#     train_tfrecords_path['fold_{}'.format(i)] = list(set(all_train_tfrecords_path) - set(val_tfrecords_path['fold_{}'.format(i)]))\n\n# for t,v in zip(train_tfrecords_path.items(),val_tfrecords_path.items()):\n#     print('\\n',t[0])\n#     for path in t[1]:\n#         print('Train :'+path)\n#     print()\n#     for path in v[1]:\n#         print('validation :'+path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Count","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# def count_data_items(filenames):\n#     # the number of data items is written in the name of the .tfrec files, \n#     # i.e. tain_fold_1-1852.tfrec = 1852 data items\n#     n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n#     return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# N_TRAIN_IMGS = {f'fold_{i}': count_data_items(train_tfrecords_path[f'fold_{i}'])\n#                 for i in range(train_fold)}\n\n# N_VAL_IMGS = {f'fold_{i}': count_data_items(val_tfrecords_path[f'fold_{i}'])\n#                 for i in range(train_fold)}\n\n# N_TEST_IMGS = count_data_items(test_tfrecords_path)\n\n# print(f\"number test image is {N_TEST_IMGS}. It is common for all folds.\")\n# print(\"-\"*75)\n# for i in range(train_fold):\n#     print(\"-\"*75)\n#     print(f\"Fold {i}: {N_TRAIN_IMGS[f'fold_{i}']} training and {N_VAL_IMGS[f'fold_{i}']} validation images.\")\n# print(\"-\"*75)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Decode Image","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# def decode_image(image_data):\n#     image = tf.image.decode_jpeg(image_data, channels=3)\n#     # convert image to floats in [0, 1] range\n#     image = tf.cast(image, tf.float32) / 255.0 \n#     # explicit size needed for TPU\n#     image = tf.reshape(image, image_shape)\n#     return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read train TFRecords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# def read_train_tfrecord(example):\n#     tfrec_format = {\n        \n#         # _bytes_feature\n#         \"image\": tf.io.FixedLenFeature([], tf.string), \n#         \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \n#         # _int64_feature\n#         \"sex_female\": tf.io.FixedLenFeature([], tf.int64),\n#         \"sex_male\": tf.io.FixedLenFeature([], tf.int64),\n#         \"sex_unknown\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_head/neck\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_lower extremity\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_oral/genital\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_palms/soles\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_torso\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_unknown\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_upper extremity\": tf.io.FixedLenFeature([], tf.int64),\n#         \"target\": tf.io.FixedLenFeature([], tf.int64),\n        \n#         # _float_feature\n#         \"age_approx\": tf.io.FixedLenFeature([], tf.float32),\n#     }\n    \n        \n        \n#     example = tf.io.parse_single_example(example, tfrec_format)\n    \n#     # image data\n#     image = decode_image(example['image']) \n    \n#     data={}\n    \n#     # _bytes_feature\n#     data['image_name']=image_name=tf.cast(example['image_name'], tf.string)\n    \n#     # integer features\n#     data['sex_female']=tf.cast(example['sex_female'], tf.int32)\n#     data['sex_male']=tf.cast(example['sex_male'], tf.int32)\n#     data['sex_unknown']=tf.cast(example['sex_unknown'], tf.int32)\n#     data['anatom_site_head/neck']=tf.cast(example['anatom_site_head/neck'], tf.int32)\n#     data['anatom_site_lower extremity']=tf.cast(example['anatom_site_lower extremity'], tf.int32)\n#     data['anatom_site_oral/genital']=tf.cast(example['anatom_site_oral/genital'], tf.int32)\n#     data['anatom_site_palms/soles']=tf.cast(example['anatom_site_palms/soles'], tf.int32)\n#     data['anatom_site_torso']=tf.cast(example['anatom_site_torso'], tf.int32)\n#     data['anatom_site_unknown']=tf.cast(example['anatom_site_unknown'], tf.int32)\n#     data['anatom_site_upper extremity']=tf.cast(example['anatom_site_upper extremity'], tf.int32)\n    \n#     # _float_feature\n#     data['age_approx']=tf.cast(example['age_approx'], tf.float32)\n\n#     target=tf.cast(example['target'], tf.int32)\n\n#     return image, target, data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read test TFRecords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# def read_test_tfrecord(example):\n#     tfrec_format = {\n        \n#         # _bytes_feature\n#         \"image\": tf.io.FixedLenFeature([], tf.string), \n#         \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \n#         # _int64_feature\n#         \"sex_female\": tf.io.FixedLenFeature([], tf.int64),\n#         \"sex_male\": tf.io.FixedLenFeature([], tf.int64),\n#         \"sex_unknown\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_head/neck\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_lower extremity\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_oral/genital\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_palms/soles\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_torso\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_unknown\": tf.io.FixedLenFeature([], tf.int64),\n#         \"anatom_site_upper extremity\": tf.io.FixedLenFeature([], tf.int64),\n        \n#         # _float_feature\n#         \"age_approx\": tf.io.FixedLenFeature([], tf.float32),\n#     }\n    \n        \n        \n#     example = tf.io.parse_single_example(example, tfrec_format)\n    \n#     # image data\n#     image = decode_image(example['image']) \n    \n#     data={}\n    \n#     # _bytes_feature\n#     data['image_name']=image_name=tf.cast(example['image_name'], tf.string)\n    \n#     # integer features\n#     data['sex_female']=tf.cast(example['sex_female'], tf.int32)\n#     data['sex_male']=tf.cast(example['sex_male'], tf.int32)\n#     data['sex_unknown']=tf.cast(example['sex_unknown'], tf.int32)\n#     data['anatom_site_head/neck']=tf.cast(example['anatom_site_head/neck'], tf.int32)\n#     data['anatom_site_lower extremity']=tf.cast(example['anatom_site_lower extremity'], tf.int32)\n#     data['anatom_site_oral/genital']=tf.cast(example['anatom_site_oral/genital'], tf.int32)\n#     data['anatom_site_palms/soles']=tf.cast(example['anatom_site_palms/soles'], tf.int32)\n#     data['anatom_site_torso']=tf.cast(example['anatom_site_torso'], tf.int32)\n#     data['anatom_site_unknown']=tf.cast(example['anatom_site_unknown'], tf.int32)\n#     data['anatom_site_upper extremity']=tf.cast(example['anatom_site_upper extremity'], tf.int32)\n    \n#     # _float_feature\n#     data['age_approx']=tf.cast(example['age_approx'], tf.float32)\n    \n#     return image, data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# def load_dataset(filenames, _set=\"train\", ordered=False):\n#     # Read from TFRecords. For optimal performance, reading from multiple files \n#     # at once and disregarding data order. Order does not matter since we will \n#     # be shuffling the data anyway.\n\n#     ignore_order = tf.data.Options()\n#     if not ordered:\n#         # disable order, increase speed\n#         ignore_order.experimental_deterministic = False\n\n#     # automatically interleaves reads from multiple files\n#     dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n#     # uses data as soon as it streams in, rather than in its original order\n#     dataset = dataset.with_options(ignore_order)\n#     # returns a dataset of (image, label) pairs if labeled=True \n#     # or (image, id) pairs if labeled=False\n#     dataset = dataset.map(read_train_tfrecord if _set == \"train\" \n#                           else read_test_tfrecord, num_parallel_calls=AUTO)\n    \n#     return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Take First-element of train-fold-1","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n\n# training_dataset = load_dataset(train_tfrecords_path['fold_0'])\n\n# # training_dataset.take(1)\n# # <TakeDataset shapes: ((512, 512, 3), (), \n# #                       {image_name: (), sex_female: (), sex_male: (), \n# #                       sex_unknown: (), anatom_site_head/neck: (), \n# #                       anatom_site_lower extremity: (), anatom_site_oral/genital: (), \n# #                       anatom_site_palms/soles: (), anatom_site_torso: (), \n# #                       anatom_site_unknown: (), anatom_site_upper extremity: (), \n# #                       age_approx: ()}), types: (tf.float32, tf.int32, \n# #                                                 {image_name: tf.string, sex_female: tf.int32, \n# #                                                  sex_male: tf.int32, sex_unknown: tf.int32, \n# #                                                  anatom_site_head/neck: tf.int32, \n# #                                                  anatom_site_lower extremity: tf.int32, \n# #                                                  anatom_site_oral/genital: tf.int32, \n# #                                                  anatom_site_palms/soles: tf.int32, \n# #                                                  anatom_site_torso: tf.int32, \n# #                                                  anatom_site_unknown: tf.int32, \n# #                                                  anatom_site_upper extremity: tf.int32, \n# #                                                  age_approx: tf.float32})>\n\n\n# target_map = {0:'benign',1:'malignant'}\n# print(\"Example of the training data:\\n\")\n\n# for image, target, data in training_dataset.take(1):\n#     print(\"The image batch size:\", image.numpy().shape)\n#     print(\"target name:\", target_map[target.numpy()])\n#     print(\"image Name :\", data['image_name'].numpy())\n#     print(\"sex_female :\", data['sex_female'].numpy())\n#     print(\"sex_male :\", data['sex_male'].numpy())\n#     print(\"sex_unknown :\", data['sex_unknown'].numpy())\n#     print(\"age_approx(Rescaled) :\", data['age_approx'].numpy())\n#     print(\"anatom_site_head/neck :\", data['anatom_site_head/neck'].numpy())\n#     print(\"anatom_site_lower extremity :\", data['anatom_site_lower extremity'].numpy())\n#     print(\"anatom_site_oral/genital :\", data['anatom_site_oral/genital'].numpy())\n#     print(\"anatom_site_palms/soles :\", data['anatom_site_palms/soles'].numpy())\n#     print(\"anatom_site_torso :\", data['anatom_site_torso'].numpy())\n#     print(\"anatom_site_unknown :\", data['anatom_site_unknown'].numpy())\n#     print(\"anatom_site_upper extremity :\", data['anatom_site_upper extremity'].numpy())\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Take First-element of validation-fold-1","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n\n# validation_dataset = load_dataset(train_tfrecords_path['fold_0'])\n\n# target_map = {0:'benign',1:'malignant'}\n# print(\"Example of the validation data:\\n\")\n\n# for image, target, data in validation_dataset.take(1):\n#     print(\"The image batch size:\", image.numpy().shape)\n#     print(\"target name:\", target_map[target.numpy()])\n#     print(\"image Name :\", data['image_name'].numpy())\n#     print(\"sex_female :\", data['sex_female'].numpy())\n#     print(\"sex_male :\", data['sex_male'].numpy())\n#     print(\"sex_unknown :\", data['sex_unknown'].numpy())\n#     print(\"age_approx(Rescaled) :\", data['age_approx'].numpy())\n#     print(\"anatom_site_head/neck :\", data['anatom_site_head/neck'].numpy())\n#     print(\"anatom_site_lower extremity :\", data['anatom_site_lower extremity'].numpy())\n#     print(\"anatom_site_oral/genital :\", data['anatom_site_oral/genital'].numpy())\n#     print(\"anatom_site_palms/soles :\", data['anatom_site_palms/soles'].numpy())\n#     print(\"anatom_site_torso :\", data['anatom_site_torso'].numpy())\n#     print(\"anatom_site_unknown :\", data['anatom_site_unknown'].numpy())\n#     print(\"anatom_site_upper extremity :\", data['anatom_site_upper extremity'].numpy())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## get-test-tfrecords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# def get_test_dataset(ordered=False):\n#     dataset = load_dataset(test_tfrecords_path, _set=\"test\", ordered=ordered)\n#     dataset = dataset.batch(BATCH_SIZE)\n#     # prefetch next batch while training (autotune prefetch buffer size)\n#     dataset = dataset.prefetch(AUTO)\n    \n#     return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get Test-data element","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n\n# test_dataset = get_test_dataset()\n\n# print(\"Examples of the test data:\")\n# for image, data in test_dataset.take(2):\n#     print(\"The image batch size:\", image.numpy().shape)\n#     print(\"image Name :\", data['image_name'].numpy())\n#     print(\"sex_female :\", data['sex_female'].numpy())\n#     print(\"sex_male :\", data['sex_male'].numpy())\n#     print(\"sex_unknown :\", data['sex_unknown'].numpy())\n#     print(\"age_approx(Rescaled) :\", data['age_approx'].numpy())\n#     print(\"anatom_site_head/neck :\", data['anatom_site_head/neck'].numpy())\n#     print(\"anatom_site_lower extremity :\", data['anatom_site_lower extremity'].numpy())\n#     print(\"anatom_site_oral/genital :\", data['anatom_site_oral/genital'].numpy())\n#     print(\"anatom_site_palms/soles :\", data['anatom_site_palms/soles'].numpy())\n#     print(\"anatom_site_torso :\", data['anatom_site_torso'].numpy())\n#     print(\"anatom_site_unknown :\", data['anatom_site_unknown'].numpy())\n#     print(\"anatom_site_upper extremity :\", data['anatom_site_upper extremity'].numpy())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot-Batch Images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Plot Batch\n# fig = plt.figure(figsize=(15,15))\n\n# for i,image in enumerate(next(iter(test_dataset.unbatch().batch(20)))[0].numpy()):\n    \n#     plt.subplot(4, 5, i+1)\n#     plt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* References\n    1. [how-to-create-tfrecords](https://www.kaggle.com/cdeotte/how-to-create-tfrecords)","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}