{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Acknowledgement\n\nIn this notebook, we follow the approach outlined by Martin Görner in [Part 1 of his Keras on TPU series](https://codelabs.developers.google.com/codelabs/keras-flowers-data/#0).","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Loading libraries","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os, sys, math\nimport tensorflow as tf\nfrom pathlib import Path\nfrom matplotlib import pyplot as plt\nfrom sklearn.preprocessing import StandardScaler\n\n# AUTO will be used in tf.data.Dataset API\nAUTO = tf.data.experimental.AUTOTUNE \n\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setting up basic parameters","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"show_files=0\n\n# if you want to see the full content of the\n# 'kaggle/input'directory set show_files=1\n\nif show_files:\n    for dirname, _, filenames in os.walk('/kaggle/input'):\n        for filename in filenames:\n            print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLDS=5 # this must be consistent with PATH_FOLDS below\nSHARDS = 4 # the number of .tfrec files in each fold\nTARGET_SIZE = [512, 512] # the desired size of the output images\nCLASSES = [b'benign', b'malignant']\n\nPATH_DATA=Path('/kaggle/input/siim-isic-melanoma-classification/')\nPATH_FOLDS=Path('/kaggle/input/siim-stratified-groupkfold-5-folds/')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loading training data","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train=pd.read_csv(PATH_DATA/'train.csv')\nprint(f\"The shape of the `train` is {train.shape}.\\n\")\nprint(f\"The columns present in `train` are {train.columns.values}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test=pd.read_csv(PATH_DATA/'test.csv')\nprint(f\"The shape of the `test` is {test.shape}.\\n\")\nprint(f\"The columns present in `test` are {test.columns.values}.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note that there is no `diagnosis` column in the test set. ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Imputing missing values\n\nThe `sex`, `age_approx`, and `anatom_site_general_challenge` columns of the training set contain missing values:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The `anatom_site_general_challenge` column of the test set contains missing values as well","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will replace the missing values for `age_approx` with the median age of the patients present in the dataset. As for the other two columns, we will mark the missing values with the word \"unknown\". ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"median_age=train['age_approx'].median()\nprint(f\"The median age of the patients in the training set is {median_age} years.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['age_approx'].fillna(median_age, inplace=True)\ntrain.fillna('unknown', inplace=True)\ntest.fillna('unknown', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"The total number of NA's after imputation in `train` is {train.isna().sum().sum()}.\")\nprint(f\"The total number of NA's after imputation in `test` is {test.isna().sum().sum()}.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### One-hot encoding for categorical variables","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"The unique values in `train`:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"The unique values of 'age_approx':\")\nprint(np.unique(train['age_approx'].values))\nprint(\"\\nThe unique values of 'sex':\")\nprint(np.unique(train['sex'].values))\nprint(\"\\nThe unique values of 'anatom_site_general_challenge':\")\nprint(np.unique(train['anatom_site_general_challenge'].values))\nprint(\"\\nThe unique values of 'diagnosis':\")\nprint(np.unique(train['diagnosis'].values))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The unique values in `test`:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"The unique values of 'age_approx':\")\nprint(np.unique(test['age_approx'].values))\nprint(\"\\nThe unique values of 'sex':\")\nprint(np.unique(test['sex'].values))\nprint(\"\\nThe unique values of 'anatom_site_general_challenge':\")\nprint(np.unique(test['anatom_site_general_challenge'].values))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Observe that the age values are all integer in both `train` and `test`. Let's cast `age_approx` into `np.uint8` format.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train['age_approx']=train['age_approx'].astype(np.uint8)\ntest['age_approx']=test['age_approx'].astype(np.uint8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Checking if `anatom_site_general_challenge` has the same set of values in `train` and `test`:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"np.equal(np.unique(test['anatom_site_general_challenge'].values),\n         np.unique(train['anatom_site_general_challenge'].values)\n        ).all()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Yes, it does. Now we will apply one-hot encoding to `sex` and `anatom_site_general_challenge`. We will not be one-hot encoding `diagnosis` since it is present only in the training set. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat([train, pd.get_dummies(train['sex'], prefix='sex')], axis=1)\ntrain = pd.concat([train, pd.get_dummies(train['anatom_site_general_challenge'], \n                                         prefix='site')], axis=1)\n# train = pd.concat([train, pd.get_dummies(train['diagnosis'], prefix='diagn')], axis=1)\n\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.concat([test, pd.get_dummies(test['sex'], prefix='sex')], axis=1)\ntest = pd.concat([test, pd.get_dummies(test['anatom_site_general_challenge'],\n                                       prefix='site')], axis=1)\n\ntest.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Scaling the age feature","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nscaler=StandardScaler()\n\ntrain['age_scaled']=scaler.fit_transform(train['age_approx'].values.reshape(-1, 1))\ntest['age_scaled']=scaler.transform(test['age_approx'].values.reshape(-1, 1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loading the fold indicies\n\nWe will use the validation and training fold indicies generated in [this kernel](https://www.kaggle.com/graf10a/siim-stratified-groupkfold-5-folds).","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_idx={fn: np.load(PATH_FOLDS/f\"train_idx_fold_{fn}.npy\") for fn in range(1, NFOLDS+1)}\nval_idx={fn: np.load(PATH_FOLDS/f\"val_idx_fold_{fn}.npy\") for fn in range(1, NFOLDS+1)}\n\nfor fn in range(1, NFOLDS+1):\n    print(\"=\"*50)\n    print(f\"Fold {fn}:\")\n    print(f\"The training set consists of {len(train_idx[fn])} elements.\")\n    print(f\"The validation set consists of {len(val_idx[fn])} elements.\")\n\n    assert len(train)==(len(train_idx[fn])+len(val_idx[fn])), \"Wrong total number of elements\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"excluded_cols=['sex', 'anatom_site_general_challenge', 'diagnosis', ]\n\ncols=[c for c in train.columns if c not in excluded_cols]\n\nprint(cols)\nprint(f\"\\nThe total number of features is {len(cols)}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_fold={fn: train.loc[val_idx[fn], cols] for fn in range(1, NFOLDS+1)}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Turning the fold data into a TF dataset\n\nOur next step is to make a set of Tensor Flow datasets, one for each fold. Each dataset will include the input data and the labels. The input data will placed in a dictionary. We can access different features by calling the dictionary with the corresponding key.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"labels={fn: train_fold[fn].pop('target') for fn in range(1, NFOLDS+1)}\ndataset0 = {fn: tf.data.Dataset.from_tensor_slices((dict(train_fold[fn]), labels[fn])) \n            for fn in range(1, NFOLDS+1)\n           }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The function below reads each J PEG image file from the disk using the filename provided in the `image_name` column of `train` or `test`. Then it turns the JPEG-encoded image into a uint8 tensor using `tf.image.decode_jpeg`.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_jpeg(data_dict, label): \n    fname=\"/kaggle/input/siim-isic-melanoma-classification/jpeg/train/\" \\\n          +data_dict['image_name']+\".jpg\"\n    bits = tf.io.read_file(fname)\n    data_dict['image'] = tf.image.decode_jpeg(bits)  \n    return data_dict, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Applying this function to each of the datasets using `map`:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset1 = {fn: dataset0[fn].map(decode_jpeg, num_parallel_calls=AUTO) for fn in range(1, NFOLDS+1)}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Resizing and cropping\n\nThe function below does resizing and cropping of the original images. We also add the original height and width to the list of features.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_and_crop_image(data, label):\n    # Resize and crop using \"fill\" algorithm:\n    # always make sure the resulting image\n    # is cut out from the source image so that\n    # it fills the TARGET_SIZE entirely with no\n    # black bars and a preserved aspect ratio.\n    w = tf.shape(data['image'])[0] \n    h = tf.shape(data['image'])[1]\n    tw = TARGET_SIZE[1]\n    th = TARGET_SIZE[0]\n    resize_crit = (w * th) / (h * tw)\n    data['image'] = tf.cond(resize_crit < 1,\n                            # if true\n                            lambda: tf.image.resize(data['image'], [w*tw/w, h*tw/w],\n                                                    method='lanczos3',\n                                                    antialias=True\n                                                   ),\n                            # if false\n                            lambda: tf.image.resize(data['image'], [w*th/h, h*th/h],\n                                                    method='lanczos3',\n                                                    antialias=True\n                                                   )\n                           )\n    nw = tf.shape(data['image'])[0]\n    nh = tf.shape(data['image'])[1]\n    data['image'] = tf.image.crop_to_bounding_box(data['image'], \n                                                  (nw - tw) // 2, \n                                                  (nh - th) // 2, \n                                                  tw, th\n                                                 )\n    return data, label, h, w","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset2 = {fn: dataset1[fn].map(resize_and_crop_image, num_parallel_calls=AUTO) \n            for fn in range(1, NFOLDS+1)}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Recompress the images\n\nGoogle Cloud Storage is capable of great throughput but has a per-file access penalty. Training on thousands of individual files will be too slow. We have to use the TFRecord format to group files together. To do that, we first need to recompress our images. The bandwidth savings outweight the decoding CPU cost.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def recompress_image(data, label, h, w):\n    data['image'] = tf.cast(data['image'], tf.uint8)\n    data['image'] = tf.image.encode_jpeg(data['image'], \n                                         #quality=100, # the default is 95% (the original images \n                                         # are already compressed, so no need to increase this \n                                         # value -- we can't create new information.)\n                                         optimize_size=True, \n                                         chroma_downsampling=False)\n    return data, label, h, w","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset3 = {fn: dataset2[fn].map(recompress_image, num_parallel_calls=AUTO) for fn in range(1, NFOLDS+1)}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Write dataset to TFRecord files ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"nb_images = {fn: len(train_fold[fn]) for fn in range(1, NFOLDS+1)}\nshard_size = {fn: math.ceil(1.0 * nb_images[fn] / SHARDS) for fn in range(1, NFOLDS+1)}\n\nfor fn in range(1, NFOLDS+1):\n    print(\"=\"*50)\n    print(f\"Fold {fn}:\")\n    print(f\"The total number of images = {nb_images[fn]}\")\n    print(f\"The number of  .tfrecord files = {SHARDS}\")\n    print(f\"The number of images in each .tfrecord file = {shard_size[fn]}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Sharding: there will be one \"batch\" of images per file","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset4 = {fn: dataset3[fn].batch(shard_size[fn]) for fn in range(1, NFOLDS+1)}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Three types of data can be stored in TFRecords: bytestrings, integers and floats. They are always stored as lists, a single data element will be a list of size 1.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytestring_feature(list_of_bytestrings):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=list_of_bytestrings))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _int_feature(list_of_ints): # int64\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=list_of_ints))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _float_feature(list_of_floats): # float32\n    return tf.train.Feature(float_list=tf.train.FloatList(value=list_of_floats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_tfrecord(tfrec_filewriter, image, image_name, patient_id, \n                benign_malignant, age, age_scaled, sex_female, sex_male, sex_unknown, \n                site_head_neck, site_lower_extremity, site_oral_genital, \n                site_palms_soles, site_torso, site_unknown, site_upper_extremity, \n                label, height, width):\n\n    feature = {\n        # bytestring features\n        \"image\": _bytestring_feature([image]), \n        \"image_name\": _bytestring_feature([image_name]),\n        \"patient_id\": _bytestring_feature([patient_id]), \n        \"benign_malignant\": _bytestring_feature([benign_malignant]),\n        # integer features\n        \"age\": _int_feature([age]),\n        \"sex_female\": _int_feature([sex_female]),        \n        \"sex_male\": _int_feature([sex_male]),\n        \"sex_unknown\": _int_feature([sex_unknown]),\n        \"site_head/neck\": _int_feature([site_head_neck]),\n        \"site_lower extremity\": _int_feature([site_lower_extremity]),\n        \"site_oral/genital\": _int_feature([site_oral_genital]),\n        \"site_palms/soles\": _int_feature([site_palms_soles]), \n        \"site_torso\": _int_feature([site_torso]), \n        \"site_unknown\": _int_feature([site_unknown]), \n        \"site_upper extremity\": _int_feature([site_upper_extremity]),\n        \"height\": _int_feature([height]),\n        \"width\": _int_feature([width]),\n        \"target\": _int_feature([label]),\n        # float features\n        \"age_scaled\": _float_feature([age_scaled]),\n    }\n    \n    return tf.train.Example(features=tf.train.Features(feature=feature))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Writing TFRecords\")\n\nfor fn in range(1, NFOLDS+1):\n    \n    print(\"=\"*50)\n    print(f\"Fold {fn} out of {NFOLDS}:\")\n    \n    for shard, (data, label, height, width) in enumerate(dataset4[fn]):\n        # batch size used as shard size here\n        shard_size = data['image'].numpy().shape[0]\n        # good practice to have the number of records in the filename\n        filename = \"fold_{}_{:02d}-{}.tfrec\".format(fn, shard, shard_size)\n\n        with tf.io.TFRecordWriter(filename) as out_file:\n            for i in range(shard_size):\n                example = to_tfrecord(out_file,\n                                      # re-compressed image: already a byte string\n                                      data['image'].numpy()[i],\n                                      data['image_name'].numpy()[i],\n                                      data['patient_id'].numpy()[i],\n                                      data['benign_malignant'].numpy()[i],\n                                      data['age_approx'].numpy()[i],\n                                      data['age_scaled'].numpy()[i],\n                                      data['sex_female'].numpy()[i],\n                                      data['sex_male'].numpy()[i],\n                                      data['sex_unknown'].numpy()[i],\n                                      data['site_head/neck'].numpy()[i],\n                                      data['site_lower extremity'].numpy()[i],\n                                      data['site_oral/genital'].numpy()[i],\n                                      data['site_palms/soles'].numpy()[i],\n                                      data['site_torso'].numpy()[i],\n                                      data['site_unknown'].numpy()[i],\n                                      data['site_upper extremity'].numpy()[i],\n                                      label.numpy()[i],\n                                      height.numpy()[i],\n                                      width.numpy()[i]\n                                     )\n\n                out_file.write(example.SerializeToString())\n\n            print(\"Wrote file {} containing {} records\".format(filename, shard_size))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}