{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport cv2\nimport gc\nimport matplotlib.pyplot as plt\n\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ndf.head()\ndf = df.sort_values(\"target\", ascending = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TOTAL_SAMPLES = 1440","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_1 = df.iloc[:534]\n\ndf_1_val = df.iloc[534:584]\n\ndf_0 = df.iloc[584:TOTAL_SAMPLES]\n\ndf_0_val = df.iloc[TOTAL_SAMPLES: TOTAL_SAMPLES+100]\ndf_0 = df_0.sample(frac = 1)\ndf_0.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.concat([df_1,df_0])\ndf_train = df_train.sample(frac=1)\n\ndf_val = pd.concat([df_1_val, df_0_val])\ndf_val = df_val.sample(frac=1)\ndf_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessing(dataset):\n\n    dataset['sex'].fillna(\"no sex\", inplace = True)\n    dataset['age_approx'].fillna(0, inplace = True)\n    dataset['anatom_site_general_challenge'].fillna(\"NA\", inplace = True)\n    dataset = dataset.replace(to_replace = ['male'], value = 0)\n    dataset = dataset.replace(to_replace = ['female'], value = 1)\n    dataset = dataset.replace(to_replace = ['no sex'], value = 2)\n    dataset = dataset.replace(to_replace = ['torso'], value = 0)\n    dataset = dataset.replace(to_replace = ['lower extremity'], value = 1)\n    dataset = dataset.replace(to_replace = ['upper extremity'], value = 2)\n    dataset = dataset.replace(to_replace = ['head/neck'], value = 3)\n    dataset = dataset.replace(to_replace = ['NA'], value = 4)\n    dataset = dataset.replace(to_replace = ['palms/soles'], value = 5)\n    dataset = dataset.replace(to_replace = ['oral/genital'], value = 6)\n    \n    return dataset\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = preprocessing(df_train)\ndf_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_val = preprocessing(df_val)\ndf_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = preprocessing(pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv'))\ndf_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if (isinstance(value, type(tf.constant(0)))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_example(image_string, sex, age_approx,anatom_site_general_challenge, target, isTrain, image_name):\n\n    if (isTrain):\n        feature = {\n            'sex': _int64_feature(sex),\n            'age_approx': _float_feature(age_approx),\n            'anatom_site_general_challenge': _int64_feature(anatom_site_general_challenge),\n            'target': _int64_feature(target),\n            'image_raw': _bytes_feature(image_string),\n            }\n    else:\n        feature = {\n            'image_name': _bytes_feature(image_name),\n            'sex': _int64_feature(sex),\n            'age_approx': _float_feature(age_approx),\n            'anatom_site_general_challenge': _int64_feature(anatom_site_general_challenge),\n            'image_raw': _bytes_feature(image_string),\n            }\n        \n    return tf.train.Example(features=tf.train.Features(feature=feature))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_HEIGHT = 300\nIMAGE_WIDTH = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 16\nSHUFFLE_SIZE = TOTAL_SAMPLES\nSTEPS_PER_EPOCH = int(TOTAL_SAMPLES/BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"record_file = '/kaggle/working/train.tfrecords'\nwith tf.io.TFRecordWriter(record_file) as writer:\n    for row in df_train.iterrows():\n#         image_string = open('/kaggle/input/siim-isic-melanoma-classification/jpeg/train/' + row[1]['image_name'] + '.jpg', 'rb').read()\n        im = cv2.imread('/kaggle/input/siim-isic-melanoma-classification/jpeg/train/' + row[1]['image_name'] + '.jpg')\n        im_resize = cv2.resize(im, (IMAGE_HEIGHT, IMAGE_WIDTH))\n        is_success, im_buf_arr = cv2.imencode(\".jpg\", im_resize)\n        image_string = im_buf_arr.tobytes()\n        tf_example = image_example(image_string,row[1]['sex'], row[1]['age_approx'],row[1]['anatom_site_general_challenge'], row[1]['target'], isTrain = True,image_name = None)\n        writer.write(tf_example.SerializeToString())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"record_file = '/kaggle/working/validation.tfrecords'\nwith tf.io.TFRecordWriter(record_file) as writer:\n    for row in df_val.iterrows():\n#         image_string = open('/kaggle/input/siim-isic-melanoma-classification/jpeg/train/' + row[1]['image_name'] + '.jpg', 'rb').read()\n        im = cv2.imread('/kaggle/input/siim-isic-melanoma-classification/jpeg/train/' + row[1]['image_name'] + '.jpg')\n        im_resize = cv2.resize(im, (IMAGE_HEIGHT, IMAGE_WIDTH))\n        is_success, im_buf_arr = cv2.imencode(\".jpg\", im_resize)\n        image_string = im_buf_arr.tobytes()\n        tf_example = image_example(image_string,row[1]['sex'], row[1]['age_approx'],row[1]['anatom_site_general_challenge'], row[1]['target'], isTrain = True,image_name = None)\n        writer.write(tf_example.SerializeToString())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _parse_function_train(proto):\n    # define your tfrecord again. Remember that you saved your image as a string.\n    keys_to_features = {'image_raw': tf.io.FixedLenFeature([], tf.string),\n                        'target': tf.io.FixedLenFeature([], tf.int64),\n                        'sex': tf.io.FixedLenFeature([], tf.int64),\n                        'age_approx': tf.io.FixedLenFeature([], tf.float32),\n                       'anatom_site_general_challenge': tf.io.FixedLenFeature([],tf.int64)}\n    \n    # Load one example\n    parsed_features = tf.io.parse_single_example(proto, keys_to_features)\n    \n    image_shape = tf.stack([IMAGE_HEIGHT,IMAGE_WIDTH,3])\n#     target_shape = tf.stack([1])\n    \n    # Turn your saved image string into an array\n    parsed_features['image_raw'] = tf.io.decode_jpeg(parsed_features['image_raw'], 3)#, fixed_length = 256*256*3)\n    \n#     parsed_features['image'] = tf.cast(parsed_features['image']/255, tf.float32)\n\n    \n    parsed_features['image_raw'] = tf.reshape(parsed_features['image_raw'], image_shape)\n    \n#     parsed_features['image_raw'] = tf.image.random_brightness(parsed_features['image_raw'],0.1)\n    \n#     parsed_features['image_raw'] = tf.image.random_contrast(parsed_features['image_raw'],0.1,0.55)\n    \n    parsed_features['image_raw'] = tf.image.random_flip_left_right(parsed_features['image_raw'])\n    \n    parsed_features['image_raw'] = tf.image.random_flip_up_down(parsed_features['image_raw'])\n    \n#     parsed_features['image_raw'] = tf.image.random_saturation(parsed_features['image_raw'], 5, 10, seed=None)\n\n#     parsed_features['image_raw'] = tf.image.adjust_saturation(parsed_features['image_raw'],10)\n    \n#     parsed_features['image_raw'] = tf.image.random_saturation(parsed_features['image_raw'], 1, 5, seed=None)\n\n    parsed_features['image_raw'] = tf.image.adjust_saturation(parsed_features['image_raw'],1)\n    \n#     parsed_features['image_raw'] = tf.image.adjust_contrast(parsed_features['image_raw'],2)\n\n    \n#     parsed_features['image_raw'] = tf.image.adjust_jpeg_quality(parsed_features['image_raw'],75)\n    \n    \n#     parsed_features[\"target\"] = tf.reshape(parsed_features['target'],target_shape)\n    \n    return parsed_features['image_raw'], parsed_features['sex'], parsed_features['age_approx'], parsed_features['anatom_site_general_challenge'],parsed_features['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dataset_fetch (filenames, isTrain):\n    \n    dataset = tf.data.TFRecordDataset(filenames)\n    \n    if(isTrain == True):\n        dataset = dataset.repeat()\n\n    dataset = dataset.map(_parse_function_train)\n\n#     else:\n#         dataset = dataset.map(_parse_function_test)\n        \n#     dataset = dataset.shuffle(SHUFFLE_SIZE)\n\n    dataset = dataset.batch(BATCH_SIZE)\n        \n#     dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n            \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from PIL import Image\ntraining_dataset = dataset_fetch('./train.tfrecords',True)\n# training_dataset = training_dataset.repeat(-1)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for element in training_dataset:\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(element[4]) #target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_index=4\n\nprint(element[4][image_index])\nplt.imshow(element[0][image_index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(element[0][image_index][:,:,1],cmap = 'gray', clim = (75,250))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(np.array(element[0][image_index][:,:,2]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}