{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Acknowledgement\n\nIn this notebook, we follow the approach outlined by Martin Görner in [Part 1 of his Keras on TPU series](https://codelabs.developers.google.com/codelabs/keras-flowers-data/#0).","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Loading libraries","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os, sys, math\nimport tensorflow as tf\nfrom pathlib import Path\nfrom matplotlib import pyplot as plt\nfrom sklearn.preprocessing import StandardScaler\n\n# AUTO will be used in tf.data.Dataset API\nAUTO = tf.data.experimental.AUTOTUNE \n\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setting up basic parameters","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"show_files=0\n\n# if you want to see the full content of the\n# 'kaggle/input'directory set show_files=1\n\nif show_files:\n    for dirname, _, filenames in os.walk('/kaggle/input'):\n        for filename in filenames:\n            print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"SHARDS = 20\nTARGET_SIZE = [512, 512]\nCLASSES = [b'benign', b'malignant']\n\nPATH_DATA=Path('/kaggle/input/siim-isic-melanoma-classification/')\nPATH_FOLDS=Path('/kaggle/input/siim-stratified-groupkfold-5-folds/')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loading training data","execution_count":null},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"train=pd.read_csv(PATH_DATA/'train.csv')\nprint(f\"The shape of the `train` is {train.shape}.\\n\")\nprint(f\"The columns present in `train` are {train.columns.values}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test=pd.read_csv(PATH_DATA/'test.csv')\nprint(f\"The shape of the `test` is {test.shape}.\\n\")\nprint(f\"The columns present in `test` are {test.columns.values}.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note that there is no `diagnosis` column in the test set. ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Imputing missing values\n\nThe `sex`, `age_approx`, and `anatom_site_general_challenge` columns of the training set contain missing values:","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"train.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The `anatom_site_general_challenge` column of the test set contains missing values as well","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"test.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will replace the missing values for `age_approx` with the median age of the patients present in the dataset. As for the other two columns, we will mark the missing values with the word \"unknown\". ","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"median_age=train['age_approx'].median()\nprint(f\"The median age of the patients in the training set is {median_age} years.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train['age_approx'].fillna(median_age, inplace=True)\ntrain.fillna('unknown', inplace=True)\ntest.fillna('unknown', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(f\"The total number of NA's after imputation in `train` is {train.isna().sum().sum()}.\")\nprint(f\"The total number of NA's after imputation in `test` is {test.isna().sum().sum()}.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### One-hot encoding for categorical variables","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"The unique values in `train`:","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"The unique values of 'age_approx':\")\nprint(np.unique(train['age_approx'].values))\nprint(\"\\nThe unique values of 'sex':\")\nprint(np.unique(train['sex'].values))\nprint(\"\\nThe unique values of 'anatom_site_general_challenge':\")\nprint(np.unique(train['anatom_site_general_challenge'].values))\nprint(\"\\nThe unique values of 'diagnosis':\")\nprint(np.unique(train['diagnosis'].values))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The unique values in `test`:","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"The unique values of 'age_approx':\")\nprint(np.unique(test['age_approx'].values))\nprint(\"\\nThe unique values of 'sex':\")\nprint(np.unique(test['sex'].values))\nprint(\"\\nThe unique values of 'anatom_site_general_challenge':\")\nprint(np.unique(test['anatom_site_general_challenge'].values))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Observe that the age values are all integer in both `train` and `test`. Let's cast `age_approx` into `np.uint8` format.","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"train['age_approx']=train['age_approx'].astype(np.uint8)\ntest['age_approx']=test['age_approx'].astype(np.uint8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Checking if `anatom_site_general_challenge` has the same set of values in `train` and `test`:","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"np.equal(np.unique(test['anatom_site_general_challenge'].values),\n         np.unique(train['anatom_site_general_challenge'].values)\n        ).all()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Yes, it does. Now we will apply one-hot encoding to `sex` and `anatom_site_general_challenge`. We will not be one-hot encoding `diagnosis` since it is present only in the training set. ","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"train = pd.concat([train, pd.get_dummies(train['sex'], prefix='sex')], axis=1)\ntrain = pd.concat([train, pd.get_dummies(train['anatom_site_general_challenge'], \n                                         prefix='site')], axis=1)\n# train = pd.concat([train, pd.get_dummies(train['diagnosis'], prefix='diagn')], axis=1)\n\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test = pd.concat([test, pd.get_dummies(test['sex'], prefix='sex')], axis=1)\ntest = pd.concat([test, pd.get_dummies(test['anatom_site_general_challenge'],\n                                       prefix='site')], axis=1)\n# the following columns is added for consistency with `train`\ntest['sex_unknown']=np.zeros(len(test))\ntest['sex_unknown']=test['sex_unknown'].astype(np.uint8)\n\ntest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pd.set_option('display.max_columns', None)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Scaling the age feature","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"%%time\n\nscaler=StandardScaler()\n\ntrain['age_scaled']=scaler.fit_transform(train['age_approx'].values.reshape(-1, 1))\ntest['age_scaled']=scaler.transform(test['age_approx'].values.reshape(-1, 1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Turning the fold data into a TF dataset","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"excluded_cols=['sex', 'anatom_site_general_challenge', 'diagnosis', ]\n\ncols=[c for c in test.columns if c not in excluded_cols]\n\nprint(cols)\nprint(f\"\\nThe total number of features is {len(cols)}.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"dataset0 = tf.data.Dataset.from_tensor_slices(dict(test[cols]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def show_instance(item, special):\n    for k, v in item.items():\n        if k not in special:\n            print(k, v.numpy())\n        else:\n            print(\"Image shape\", v.numpy().shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def show_ds(ds, n=1, special=['image']):\n    for item in ds.take(n):\n        show_instance(item, special)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"show_ds(dataset0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def decode_jpeg(data_dict): \n    fname=\"/kaggle/input/siim-isic-melanoma-classification/jpeg/test/\" \\\n          +data_dict['image_name']+\".jpg\"\n    bits = tf.io.read_file(fname)\n    data_dict['image'] = tf.image.decode_jpeg(bits)  \n    return data_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"dataset1 = dataset0.map(decode_jpeg, num_parallel_calls=AUTO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"show_ds(dataset1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualization function","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"def show_9(dataset):\n    plt.figure(figsize=(13,13))\n    subplot=331\n    i=0\n    for data in dataset:  \n        i+=1\n        plt.subplot(subplot)\n        plt.axis('off')\n        plt.imshow(data['image'].numpy().astype(np.uint8))\n        subplot += 1\n        if i==9:\n            break\n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"show_9(dataset1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Resizing and cropping","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"def resize_and_crop_image(data):\n    # Resize and crop using \"fill\" algorithm:\n    # always make sure the resulting image\n    # is cut out from the source image so that\n    # it fills the TARGET_SIZE entirely with no\n    # black bars and a preserved aspect ratio.\n    w = tf.shape(data['image'])[0]\n    h = tf.shape(data['image'])[1]\n    tw = TARGET_SIZE[1]\n    th = TARGET_SIZE[0]\n    resize_crit = (w * th) / (h * tw)\n    data['image'] = tf.cond(resize_crit < 1,\n                            # if true\n                            lambda: tf.image.resize(data['image'], [w*tw/w, h*tw/w],\n                                                    method='lanczos3',\n                                                    antialias=True\n                                                   ),\n                            # if false\n                            lambda: tf.image.resize(data['image'], [w*th/h, h*th/h],\n                                                    method='lanczos3',\n                                                    antialias=True\n                                                   )\n                           )\n    nw = tf.shape(data['image'])[0]\n    nh = tf.shape(data['image'])[1]\n    data['image'] = tf.image.crop_to_bounding_box(data['image'], \n                                                  (nw - tw) // 2, \n                                                  (nh - th) // 2, \n                                                  tw, th\n                                                 )\n    return data, h, w","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"dataset2 = dataset1.map(resize_and_crop_image, num_parallel_calls=AUTO)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Redefine our plotting finction to account for the new height and width features (alternatively, you can just add these features to the `data` dictionary).","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"def show_9(dataset):\n    plt.figure(figsize=(13,13))\n    subplot=331\n    i=0\n    for data, h, w in dataset:  \n        i+=1\n        plt.subplot(subplot)\n        plt.axis('off')\n        plt.imshow(data['image'].numpy().astype(np.uint8))\n        subplot += 1\n        if i==9:\n            break\n    plt.tight_layout()\n    plt.subplots_adjust(wspace=0.1, hspace=0.1)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"show_9(dataset2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Speed test: too slow\n\nGoogle Cloud Storage is capable of great throughput but has a per-file access penalty. Run the cell below and see that throughput is around 5 images per second (at least this was the speed at the time of writing this notebook).","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"%%time\n\ndisplay_dataset = dataset2.batch(10)\nfor item, h, w in display_dataset.take(10):\n    print(f\"Image batch shape {item['image'].numpy().shape}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Recompress the images\n\nAs we just saw, working with thousands of individual files will be too slow. We have to use the TFRecord format to group files together. To do that, we first need to recompress our images. The bandwidth savings outweight the decoding CPU cost. The bandwidth savings outweight the decoding CPU cost.","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"def recompress_image(data, h, w):\n\n    data['image'] = tf.cast(data['image'], tf.uint8)\n    data['image'] = tf.image.encode_jpeg(data['image'], \n                                         #quality=100,\n                                         optimize_size=True, \n                                         chroma_downsampling=False)\n    return data, h, w","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"dataset3 = dataset2.map(recompress_image, num_parallel_calls=AUTO)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Write dataset to TFRecord files ","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"nb_images = len(test)\nshard_size = math.ceil(1.0 * nb_images / SHARDS)\n\nprint(f\"The total number of images = {nb_images}\")\nprint(f\"The number of  .tfrecord files = {SHARDS}\")\nprint(f\"The number of images in each .tfrecord file = {shard_size}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Sharding: there will be one \"batch\" of images per file","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"dataset4 = dataset3.batch(shard_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Three types of data can be stored in TFRecords: bytestrings, integers and floats. They are always stored as lists, a single data element will be a list of size 1.","execution_count":null},{"metadata":{"trusted":false},"cell_type":"code","source":"def _bytestring_feature(list_of_bytestrings):\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=list_of_bytestrings))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def _int_feature(list_of_ints): # int64\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=list_of_ints))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def _float_feature(list_of_floats): # float32\n    return tf.train.Feature(float_list=tf.train.FloatList(value=list_of_floats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def to_tfrecord(tfrec_filewriter, image, image_name, patient_id, \n                age, age_scaled, sex_female, sex_male, sex_unknown, \n                site_head_neck, site_lower_extremity, site_oral_genital, \n                site_palms_soles, site_torso, site_unknown, site_upper_extremity, \n                height, width):\n\n    feature = {\n        # bytestring features\n        \"image\": _bytestring_feature([image]), \n        \"image_name\": _bytestring_feature([image_name]),\n        \"patient_id\": _bytestring_feature([patient_id]), \n        # integer features\n        \"age\": _int_feature([age]),\n        \"sex_female\": _int_feature([sex_female]),        \n        \"sex_male\": _int_feature([sex_male]),\n        \"sex_unknown\": _int_feature([sex_unknown]),\n        \"site_head/neck\": _int_feature([site_head_neck]),\n        \"site_lower extremity\": _int_feature([site_lower_extremity]),\n        \"site_oral/genital\": _int_feature([site_oral_genital]),\n        \"site_palms/soles\": _int_feature([site_palms_soles]), \n        \"site_torso\": _int_feature([site_torso]), \n        \"site_unknown\": _int_feature([site_unknown]), \n        \"site_upper extremity\": _int_feature([site_upper_extremity]),\n        \"height\": _int_feature([height]),\n        \"width\": _int_feature([width]),\n        # float features\n        \"age_scaled\": _float_feature([age_scaled]),\n    }\n    \n    return tf.train.Example(features=tf.train.Features(feature=feature))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"Writing TFRecords\")\nfor shard, (data, height, width) in enumerate(dataset4):\n    \n#     if shard not in range(SHARDS//2*(N-1), SHARDS//2*N):\n#         continue\n    # batch size used as shard size here\n    shard_size = data['image'].numpy().shape[0]\n    # good practice to have the number of records in the filename\n    filename = \"{:02d}-{}.tfrec\".format(shard, shard_size)\n\n    with tf.io.TFRecordWriter(filename) as out_file:\n        for i in range(shard_size):\n            example = to_tfrecord(out_file,\n                                  # re-compressed image: already a byte string\n                                  data['image'].numpy()[i],\n                                  data['image_name'].numpy()[i],\n                                  data['patient_id'].numpy()[i],\n                                  data['age_approx'].numpy()[i],\n                                  data['age_scaled'].numpy()[i],\n                                  data['sex_female'].numpy()[i],\n                                  data['sex_male'].numpy()[i],\n                                  data['sex_unknown'].numpy()[i],\n                                  data['site_head/neck'].numpy()[i],\n                                  data['site_lower extremity'].numpy()[i],\n                                  data['site_oral/genital'].numpy()[i],\n                                  data['site_palms/soles'].numpy()[i],\n                                  data['site_torso'].numpy()[i],\n                                  data['site_unknown'].numpy()[i],\n                                  data['site_upper extremity'].numpy()[i],\n                                  height.numpy()[i],\n                                  width.numpy()[i]\n                                 )\n\n            out_file.write(example.SerializeToString())\n\n    print(\"Wrote file {} containing {} records\".format(filename, shard_size))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}