{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#import modules\n\n#PyTorch modules\nimport torch\nimport torchvision  \nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.transforms as transforms       # Transformations we can perform on our dataset\n#import torch.nn as nn                             # for all neural network module, nn.Linear, nn.Conv2d, Loss functions, BatchNorm, etc\n#import torch.optim as optim                       # for optimization algos (ADAM, SGD etc.)\n#import torch.nn.functional as F                    # for functions with no params\n\n\nimport matplotlib.pyplot as plt                   # to display images\nimport matplotlib.image as mpimg\nfrom PIL import Image\nimport seaborn as sns\nimport numpy as np                      \nimport pandas as pd\nimport cv2\nimport os\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Set Color Palettes for the notebook\ncolors_nude = ['#e0798c','#65365a','#da8886','#cfc4c4','#dfd7ca']\n#sns.palplot(sns.color_palette(colors_nude))","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:47:51.426302Z","iopub.execute_input":"2022-03-20T18:47:51.426652Z","iopub.status.idle":"2022-03-20T18:47:54.04555Z","shell.execute_reply.started":"2022-03-20T18:47:51.426541Z","shell.execute_reply":"2022-03-20T18:47:54.044815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Analyze and sample dataset","metadata":{}},{"cell_type":"code","source":"directory = '../input/siim-isic-melanoma-classification/'\ntrain_csv = pd.read_csv(directory + \"train.csv\")\ntest_csv = pd.read_csv(directory + \"test.csv\")\nprint('Train set has {:,} rows and Test set has {:,} rows.'.format(len(train_csv), len(test_csv)))\n#train_csv.head(5)\n#test_csv.head(5)\n# Train and test data info\nprint('Train Set')\nprint(train_csv.info())\nprint('-------------')\nprint('Test Set')\nprint(test_csv.info())\n","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:47:54.048845Z","iopub.execute_input":"2022-03-20T18:47:54.049045Z","iopub.status.idle":"2022-03-20T18:47:54.187994Z","shell.execute_reply.started":"2022-03-20T18:47:54.04902Z","shell.execute_reply":"2022-03-20T18:47:54.187301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv['benign_malignant'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:47:54.189231Z","iopub.execute_input":"2022-03-20T18:47:54.189642Z","iopub.status.idle":"2022-03-20T18:47:54.202254Z","shell.execute_reply.started":"2022-03-20T18:47:54.18959Z","shell.execute_reply":"2022-03-20T18:47:54.20134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function for plotting samples\ndef plot_samples(samples):  \n    fig, axes = plt.subplots(nrows=4, ncols=5, figsize=(32,16))\n    for i in range(len(samples)):\n        image = mpimg.imread(samples[i])\n        ax = axes[i//5][i%5]\n        ax.imshow(image)\n        if i<10: # first 10 files\n            ax.set_title(\"Benign\", fontsize=20)\n        else:\n            ax.set_title(\"Malignant\", fontsize=20)\n        ax.axis('off')\n        \nsample_imgs = []\nbenign_sample = train_csv[train_csv['benign_malignant'] == 'benign']['image_name'][:10]\nfor f in benign_sample: \n  sample_imgs.append(directory + '/jpeg/train/'+f+'.jpg')\n# get 10 malignant image files \nmalignant_sample = train_csv[train_csv['benign_malignant'] == 'malignant']['image_name'][:10]\nfor f in malignant_sample: \n  sample_imgs.append(directory + '/jpeg/train/'+f+'.jpg')\n\nplot_samples(sample_imgs)\nplt.suptitle('Melanoma Samples', fontsize=30)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:47:54.20472Z","iopub.execute_input":"2022-03-20T18:47:54.205017Z","iopub.status.idle":"2022-03-20T18:48:26.25642Z","shell.execute_reply.started":"2022-03-20T18:47:54.204967Z","shell.execute_reply":"2022-03-20T18:48:26.255674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Add image path to dataframes","metadata":{}},{"cell_type":"code","source":"#=== JPEG ===\n# Create the paths\npath_train = directory + 'jpeg/train/' + train_csv['image_name'] + '.jpg'\npath_test = directory + 'jpeg/test/' + test_csv['image_name'] + '.jpg'\n# Append to the original dataframes\ntrain_csv['path_jpeg'] = path_train\ntest_csv['path_jpeg'] = path_test","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:48:26.257495Z","iopub.execute_input":"2022-03-20T18:48:26.257763Z","iopub.status.idle":"2022-03-20T18:48:26.284344Z","shell.execute_reply.started":"2022-03-20T18:48:26.257713Z","shell.execute_reply":"2022-03-20T18:48:26.283763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we will analyze the image sizes, we will use a sample size of 200 images.\nWe should expect images to have various sizes as they have different sources.\nThis will need to be addressed during data Augmentation","metadata":{}},{"cell_type":"code","source":"train_sample_sizes = []\n\nfor k, path in enumerate(train_csv['path_jpeg']):\n    image = Image.open(path)\n    train_sample_sizes.append(image.size)\n    \n    if k >= 200: break\n        \ntrain_sample_sizes = pd.DataFrame(data = train_sample_sizes, columns = ['H', 'W'], dtype='object')\ntrain_sample_sizes['Size'] = '[' + train_sample_sizes['H'].astype(str) + ', ' + train_sample_sizes['W'].astype(str) + ']'","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:48:26.285572Z","iopub.execute_input":"2022-03-20T18:48:26.285993Z","iopub.status.idle":"2022-03-20T18:48:30.063715Z","shell.execute_reply.started":"2022-03-20T18:48:26.285959Z","shell.execute_reply":"2022-03-20T18:48:30.062916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (16, 6))\na = sns.countplot(train_sample_sizes['Size'], palette=colors_nude)\nfor p in a.patches:\n    a.annotate(format(p.get_height(), ','), \n           (p.get_x() + p.get_width() / 2., \n            p.get_height()), ha = 'center', va = 'center', \n           xytext = (0, 4), textcoords = 'offset points')\n    \nplt.title('200 Sampled Images Shapes', fontsize=16)\nsns.despine(left=True, bottom=True);","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:48:30.066442Z","iopub.execute_input":"2022-03-20T18:48:30.067035Z","iopub.status.idle":"2022-03-20T18:48:30.306063Z","shell.execute_reply.started":"2022-03-20T18:48:30.067003Z","shell.execute_reply":"2022-03-20T18:48:30.305411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**What is Data Augmentation?**\n\nIt consists of moving, rotating, cropping, flipping, changing color/brightness/hue in order to alter the orioginal image. It is helpful in Overfitting, as the model learns not only 1 aspect of the image, but multiple (a dog can be standing up straight, or funny upside down, in a b&w image etc).\n\nWe will be using Torchvision.transforms for the data augmentation step. It works well with PyTorch for our purposes","metadata":{}},{"cell_type":"code","source":"image_list = train_csv.sample(12)['path_jpeg']\nimage_list = image_list.reset_index()['path_jpeg']\n\n# Show the sample\nplt.figure(figsize=(16,6))\nplt.suptitle(\"Sample images\", fontsize = 16)\n    \nfor k, path in enumerate(image_list):\n    image = mpimg.imread(path)\n    plt.subplot(2, 6, k+1)\n    plt.imshow(image)\n    #plt.axis('on')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:48:30.307441Z","iopub.execute_input":"2022-03-20T18:48:30.308182Z","iopub.status.idle":"2022-03-20T18:48:47.615227Z","shell.execute_reply.started":"2022-03-20T18:48:30.308144Z","shell.execute_reply":"2022-03-20T18:48:47.614618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Class to create PyTorch Dataset Object\nclass create_dataset(Dataset):\n    def __init__(self, image_list, transforms=None, no_path=False):\n        self.image_list = image_list\n        self.transforms = transforms\n        self.no_path = no_path\n    \n    # To get item's length\n    def __len__(self):\n        return (len(self.image_list))\n    \n    # For indexing\n    def __getitem__(self, i):\n        # Read in image\n        if (self.no_path):\n            image = self.image_list[i]\n        else:\n            image = plt.imread(self.image_list[i])\n        image = Image.fromarray(image).convert('RGB')        \n        image = np.asarray(image).astype(np.uint8)\n        if self.transforms is not None:\n            image = self.transforms(image)\n            \n        return torch.tensor(image, dtype=torch.float)\n  \n# Predefined Show Images Function\ndef show_transform(image, title=\"Default\"):\n    plt.figure(figsize=(16,6))\n    plt.suptitle(title, fontsize = 16)\n    # Unnormalize\n    image = image / 2 + 0.5  \n    npimg = image.numpy()\n    npimg = np.clip(npimg, 0., 1.)\n    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:48:47.616404Z","iopub.execute_input":"2022-03-20T18:48:47.61697Z","iopub.status.idle":"2022-03-20T18:48:47.628461Z","shell.execute_reply.started":"2022-03-20T18:48:47.616932Z","shell.execute_reply":"2022-03-20T18:48:47.627826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Transforms to crop images, change brightness, contrast, and saturation, convert it to grayscale, and flip it vertically","metadata":{}},{"cell_type":"code","source":"# Crop Transform\ncrop_transform = transforms.Compose([\n     transforms.ToPILImage(),\n     transforms.Resize((300, 300)),\n     transforms.CenterCrop((100, 100)),\n     transforms.ToTensor(),\n     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n     ])\n\n# Color Jitter Transform\n# Randomly change the brightness, contrast and saturation of an image\ncolor_jitter_transform = transforms.Compose([\n     transforms.ToPILImage(),\n     transforms.Resize((300, 300)),\n     transforms.ColorJitter(brightness=0.7, contrast=0.7, saturation=0.7, hue=0.5),\n     transforms.ToTensor(),\n     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n     ])\n\n# Grayscale Transform\n# Randomly convert image to grayscale with a probability of p (default 0.1)\ngrayscale_transform = transforms.Compose([\n     transforms.ToPILImage(),\n     transforms.Resize((300, 300)),\n     transforms.RandomGrayscale(p=0.7),\n     transforms.ToTensor(),\n     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n     ])\n\n# Vertical Flip Transform\n# Vertically flip the given PIL Image randomly with a given probability\nvertical_flip_transform = transforms.Compose([\n     transforms.ToPILImage(),\n     transforms.Resize((300, 300)),\n     transforms.RandomVerticalFlip(p=0.7),\n     transforms.ToTensor(),\n     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n     ])\n\n# COmbination of the previous transforms combined\ncombination_tranform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.Resize((300, 300)),\n    transforms.CenterCrop((200, 200)),\n    transforms.ColorJitter(brightness=0.5, contrast=0.3, saturation=0.7, hue=0.5),\n    transforms.RandomGrayscale(p=0.4),\n    transforms.RandomVerticalFlip(p=0.7),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n    ])","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:48:47.63079Z","iopub.execute_input":"2022-03-20T18:48:47.631348Z","iopub.status.idle":"2022-03-20T18:48:47.644637Z","shell.execute_reply.started":"2022-03-20T18:48:47.631304Z","shell.execute_reply":"2022-03-20T18:48:47.643938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hair Removal","metadata":{}},{"cell_type":"code","source":"def hair_remove(image):\n    # convert image to grayScale\n    grayScale = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)\n\n    # kernel for morphologyEx\n    kernel = cv2.getStructuringElement(1,(17,17))\n\n    # apply MORPH_BLACKHAT to grayScale image\n    blackhat = cv2.morphologyEx(grayScale, cv2.MORPH_BLACKHAT, kernel)\n\n    # apply thresholding to blackhat\n    _,threshold = cv2.threshold(blackhat,10,255,cv2.THRESH_BINARY)\n\n    # inpaint with original image and threshold image\n    final_image = cv2.inpaint(image,threshold,1,cv2.INPAINT_TELEA)\n\n    return final_image","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:48:47.645541Z","iopub.execute_input":"2022-03-20T18:48:47.645777Z","iopub.status.idle":"2022-03-20T18:48:47.657127Z","shell.execute_reply.started":"2022-03-20T18:48:47.645745Z","shell.execute_reply":"2022-03-20T18:48:47.656389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hairy_photos = train_csv[train_csv[\"sex\"] == \"male\"].reset_index().iloc[[12, 14, 17, 22, 33, 34]]\nimage_list = hairy_photos['path_jpeg']\nimage_list = image_list.reset_index()['path_jpeg']\nplt.figure(figsize=(16,3))\nplt.suptitle(\"Original Hairy Images\", fontsize = 16)\nfor k, path in enumerate(image_list):\n    image = mpimg.imread(path)\n    image = cv2.resize(image,(300, 300))\n    plt.subplot(1, 6, k+1)\n    plt.imshow(image)\n    plt.axis('off')\n    \nplt.figure(figsize=(16,3))\nplt.suptitle(\"Non Hairy Images\", fontsize = 16)  \nfor k, path in enumerate(image_list):\n    image = mpimg.imread(path)\n    image = cv2.resize(image,(300, 300))\n    image = hair_remove(image)   \n    plt.subplot(1, 6, k+1)\n    plt.imshow(image)\n    plt.axis('off')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:48:47.658636Z","iopub.execute_input":"2022-03-20T18:48:47.659233Z","iopub.status.idle":"2022-03-20T18:48:53.242299Z","shell.execute_reply.started":"2022-03-20T18:48:47.659196Z","shell.execute_reply":"2022-03-20T18:48:53.241655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Show sample images and how the individual transform affect them, as well as the combination of all 4","metadata":{}},{"cell_type":"code","source":"# Create the dataset and apply transform\npytorch_crop_dataset = create_dataset(image_list=image_list, transforms=crop_transform, no_path=False)\npytorch_color_dataset = create_dataset(image_list=image_list, transforms=color_jitter_transform, no_path=False)\npytorch_bw_dataset = create_dataset(image_list=image_list, transforms=grayscale_transform, no_path=False)\npytorch_vertical_flip_dataset = create_dataset(image_list=image_list, transforms=vertical_flip_transform, no_path=False)\npytorch_combination_dataset = create_dataset(image_list=image_list, transforms=combination_tranform, no_path=False)\n\n#pytorch_dataloader = DataLoader(dataset=pytorch_dataset, batch_size=12, shuffle=True)\npytorch_crop_dataloader = DataLoader(dataset=pytorch_crop_dataset, batch_size=12)\npytorch_color_dataloader = DataLoader(dataset=pytorch_color_dataset, batch_size=12)\npytorch_bw_dataloader = DataLoader(dataset=pytorch_bw_dataset, batch_size=12)\npytorch_vertical_flip_dataloader = DataLoader(dataset=pytorch_vertical_flip_dataset, batch_size=12)\npytorch_combination_dataloader = DataLoader(dataset=pytorch_combination_dataset, batch_size=12)\n\n# Select the data\ncrop_images = next(iter(pytorch_crop_dataloader))\ncolor_images = next(iter(pytorch_color_dataloader))\nbw_images = next(iter(pytorch_bw_dataloader))\nvertical_flip_images = next(iter(pytorch_vertical_flip_dataloader))\ncombination_images = next(iter(pytorch_combination_dataloader))\n \n# show original image and applied transform using show_transform()\nplt.figure(figsize=(16,6))\nplt.suptitle(\"Original View\", fontsize = 16)\nfor k, path in enumerate(image_list):\n    image = mpimg.imread(path)\n    plt.subplot(2, 6, k+1)\n    plt.imshow(image)\n    plt.axis('off')\nshow_transform(torchvision.utils.make_grid(crop_images, nrow=6), title=\"Crop\")\nshow_transform(torchvision.utils.make_grid(color_images, nrow=6), title=\"Color Jitter\")\nshow_transform(torchvision.utils.make_grid(bw_images, nrow=6), title=\"BW\")\nshow_transform(torchvision.utils.make_grid(vertical_flip_images, nrow=6), title=\"Vertical Flip\")\nshow_transform(torchvision.utils.make_grid(combination_images, nrow=6), title=\"Combination tranforms\")","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:48:53.243398Z","iopub.execute_input":"2022-03-20T18:48:53.243757Z","iopub.status.idle":"2022-03-20T18:49:22.940109Z","shell.execute_reply.started":"2022-03-20T18:48:53.243724Z","shell.execute_reply":"2022-03-20T18:49:22.939448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Attempt to remove the hair AND run combination tranform on sample images","metadata":{}},{"cell_type":"code","source":"test_img = train_csv[train_csv[\"sex\"] == \"male\"].reset_index().iloc[[12,34,65,89,86,92]]\ntest_img_path = test_img['path_jpeg']\nplt.figure(figsize=(16,6))\nplt.suptitle(\"Sample image\", fontsize = 16)\n\ncln_imgs = [];\n\nfor k, path in enumerate(test_img_path):\n    image = mpimg.imread(path)\n    image = cv2.resize(image,(300, 300))\n    plt.subplot(1, 6, k+1)\n    plt.imshow(image)\n    plt.axis('off')\n    image = hair_remove(image)        \n    image = combination_tranform(image)\n    #turn image into tensor, so cln_imgs becomes list of tensors\n    cln_imgs.append(image)\n    \n","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:49:22.941636Z","iopub.execute_input":"2022-03-20T18:49:22.942173Z","iopub.status.idle":"2022-03-20T18:49:25.656583Z","shell.execute_reply.started":"2022-03-20T18:49:22.942136Z","shell.execute_reply":"2022-03-20T18:49:25.65599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_transform(torchvision.utils.make_grid(cln_imgs, nrow=6), title=\"No hair and augmented\")","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:49:25.657622Z","iopub.execute_input":"2022-03-20T18:49:25.65796Z","iopub.status.idle":"2022-03-20T18:49:25.941036Z","shell.execute_reply.started":"2022-03-20T18:49:25.657928Z","shell.execute_reply":"2022-03-20T18:49:25.940337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Building our model","metadata":{}},{"cell_type":"code","source":"import re\nimport os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom functools import partial\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nimport tempfile\nimport matplotlib.pyplot as plt\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Device:', tpu.master())\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    strategy = tf.distribute.get_strategy()\nprint('Number of replicas:', strategy.num_replicas_in_sync)\n    \nprint(tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:57:30.558064Z","iopub.execute_input":"2022-03-20T18:57:30.558333Z","iopub.status.idle":"2022-03-20T18:57:34.660045Z","shell.execute_reply.started":"2022-03-20T18:57:30.558303Z","shell.execute_reply":"2022-03-20T18:57:34.659286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def connect_to_tpu(tpu_address: str = None):\n    if tpu_address is not None:  # When using GCP\n        cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(\n            tpu=tpu_address)\n        if tpu_address not in (\"\", \"local\"):\n            tf.config.experimental_connect_to_cluster(cluster_resolver)\n        tf.tpu.experimental.initialize_tpu_system(cluster_resolver)\n        strategy = tf.distribute.experimental.TPUStrategy(cluster_resolver)\n        print(\"Running on TPU \", cluster_resolver.master())\n        print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n        return cluster_resolver, strategy\n    else:                           # When using Colab or Kaggle\n        try:\n            cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n            strategy = tf.distribute.experimental.TPUStrategy(cluster_resolver)\n            print(\"Running on TPU \", cluster_resolver.master())\n            print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n            return cluster_resolver, strategy\n        except:\n            print(\"WARNING: No TPU detected.\")\n            mirrored_strategy = tf.distribute.MirroredStrategy()\n            return None, mirrored_strategy","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:20:21.181247Z","iopub.execute_input":"2022-03-20T18:20:21.181848Z","iopub.status.idle":"2022-03-20T18:20:21.189748Z","shell.execute_reply.started":"2022-03-20T18:20:21.181814Z","shell.execute_reply":"2022-03-20T18:20:21.1887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"connect_to_tpu();","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:20:30.48232Z","iopub.execute_input":"2022-03-20T18:20:30.482885Z","iopub.status.idle":"2022-03-20T18:20:38.791911Z","shell.execute_reply.started":"2022-03-20T18:20:30.482851Z","shell.execute_reply":"2022-03-20T18:20:38.791135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# GPU\nimport tensorflow as tf\nprint(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:57:41.958524Z","iopub.execute_input":"2022-03-20T18:57:41.958919Z","iopub.status.idle":"2022-03-20T18:57:42.013041Z","shell.execute_reply.started":"2022-03-20T18:57:41.958885Z","shell.execute_reply":"2022-03-20T18:57:42.011039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device_name = tf.test.gpu_device_name()\nif \"GPU\" not in device_name:\n    print(\"GPU device not found\")\nprint('Found GPU at: {}'.format(device_name))","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:57:52.822306Z","iopub.execute_input":"2022-03-20T18:57:52.823503Z","iopub.status.idle":"2022-03-20T18:57:52.835954Z","shell.execute_reply.started":"2022-03-20T18:57:52.823472Z","shell.execute_reply":"2022-03-20T18:57:52.835247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\n\n\nGCS_PATH = KaggleDatasets().get_gcs_path()\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nIMAGE_SIZE = [1024, 1024]\nIMAGE_RESIZE = [256, 256]","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:58:02.72048Z","iopub.execute_input":"2022-03-20T18:58:02.721023Z","iopub.status.idle":"2022-03-20T18:58:03.276115Z","shell.execute_reply.started":"2022-03-20T18:58:02.720982Z","shell.execute_reply":"2022-03-20T18:58:03.275368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAINING_FILENAMES, VALID_FILENAMES = train_test_split(\n    tf.io.gfile.glob(GCS_PATH + '/tfrecords/train*.tfrec'),\n    test_size=0.1, random_state=5\n)\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/tfrecords/test*.tfrec')\nprint('Train TFRecord Files:', len(TRAINING_FILENAMES))\nprint('Validation TFRecord Files:', len(VALID_FILENAMES))\nprint('Test TFRecord Files:', len(TEST_FILENAMES))","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:58:06.747334Z","iopub.execute_input":"2022-03-20T18:58:06.74768Z","iopub.status.idle":"2022-03-20T18:58:07.6695Z","shell.execute_reply.started":"2022-03-20T18:58:06.747632Z","shell.execute_reply":"2022-03-20T18:58:07.668792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data loading methods","metadata":{}},{"cell_type":"code","source":"# Function to decode image\ndef decode_image(image):\n    image = tf.image.decode_jpeg(image, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\n# Read tfrecord\ndef read_tfrecord(example, labeled):\n    tfrecord_format = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"target\": tf.io.FixedLenFeature([], tf.int64)\n    } if labeled else {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }\n    example = tf.io.parse_single_example(example, tfrecord_format)\n    image = decode_image(example['image'])\n    if labeled:\n        label = tf.cast(example['target'], tf.int32)\n        return image, label\n    idnum = example['image_name']\n    return image, idnum\n\n#Load dataset\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTOTUNE) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(partial(read_tfrecord, labeled=labeled), num_parallel_calls=AUTOTUNE)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\n#TF data augmentation\ndef augmentation_pipeline(image, label):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.resize(image, IMAGE_RESIZE)\n    return image, label\n##added\ndef resize_augmentation_pipeline(image, label):\n    image = tf.image.resize(image, IMAGE_RESIZE)\n    return image, label\n\n#Get datasets\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(augmentation_pipeline, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALID_FILENAMES, labeled=True, ordered=ordered)\n    #added\n    dataset = dataset.map(resize_augmentation_pipeline, num_parallel_calls=AUTOTUNE)\n    ##\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    #added\n    dataset = dataset.map(resize_augmentation_pipeline, num_parallel_calls=AUTOTUNE)\n    ##\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\n# Count items in dataset\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:58:13.785893Z","iopub.execute_input":"2022-03-20T18:58:13.786319Z","iopub.status.idle":"2022-03-20T18:58:13.805409Z","shell.execute_reply.started":"2022-03-20T18:58:13.786287Z","shell.execute_reply":"2022-03-20T18:58:13.804568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALID_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nprint(\n    'Dataset: {} training images, {} validation images, {} unlabeled test images'.format(\n        NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES\n    )\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:58:19.031347Z","iopub.execute_input":"2022-03-20T18:58:19.032122Z","iopub.status.idle":"2022-03-20T18:58:19.037942Z","shell.execute_reply.started":"2022-03-20T18:58:19.032079Z","shell.execute_reply":"2022-03-20T18:58:19.037126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = get_training_dataset()\nvalid_dataset = get_validation_dataset()\n\nimage_batch, label_batch = next(iter(train_dataset))\n\ndef show_batch(image_batch, label_batch):\n    plt.figure(figsize=(10,10))\n    for n in range(25):\n        ax = plt.subplot(5,5,n+1)\n        plt.imshow(image_batch[n])\n        if label_batch[n]:\n            plt.title(\"MALIGNANT\")\n        else:\n            plt.title(\"BENIGN\")\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:58:22.137987Z","iopub.execute_input":"2022-03-20T18:58:22.138889Z","iopub.status.idle":"2022-03-20T18:58:56.257242Z","shell.execute_reply.started":"2022-03-20T18:58:22.138838Z","shell.execute_reply":"2022-03-20T18:58:56.256484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define learning rate; it will update for eeach epoch for increased efficiency and monitor model convergence\ndef exponential_decay(lr0, s):\n    def exponential_decay_fn(epoch):\n        return lr0 * 0.1 **(epoch / s)\n    return exponential_decay_fn\n\nexponential_decay_fn = exponential_decay(0.01, 20)\n\nlr_scheduler = tf.keras.callbacks.LearningRateScheduler(exponential_decay_fn)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T18:59:54.34033Z","iopub.execute_input":"2022-03-20T18:59:54.340633Z","iopub.status.idle":"2022-03-20T18:59:55.546855Z","shell.execute_reply.started":"2022-03-20T18:59:54.340582Z","shell.execute_reply":"2022-03-20T18:59:55.546077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_model(output_bias = None, metrics = None):    \n    if output_bias is not None:\n        output_bias = tf.keras.initializers.Constant(output_bias)\n        \n    base_model = tf.keras.applications.Xception(input_shape=(*IMAGE_RESIZE, 3),\n                                                include_top=False,\n                                                weights='imagenet')\n    \n    base_model.trainable = False\n    \n    model = tf.keras.Sequential([\n        #tf.keras.layers.Reshape((256,256,3)),\n        base_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(8, activation='relu'),\n        tf.keras.layers.Dense(1, activation='sigmoid',\n                              bias_initializer=output_bias)\n    ])\n    \n    model.compile(optimizer='adam',\n                  loss='binary_crossentropy',\n                  metrics=metrics)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:00:24.147777Z","iopub.execute_input":"2022-03-20T19:00:24.148295Z","iopub.status.idle":"2022-03-20T19:00:24.154851Z","shell.execute_reply.started":"2022-03-20T19:00:24.148258Z","shell.execute_reply":"2022-03-20T19:00:24.153786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALID_STEPS = NUM_VALIDATION_IMAGES // BATCH_SIZE","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:00:05.88734Z","iopub.execute_input":"2022-03-20T19:00:05.887793Z","iopub.status.idle":"2022-03-20T19:00:05.912773Z","shell.execute_reply.started":"2022-03-20T19:00:05.887751Z","shell.execute_reply":"2022-03-20T19:00:05.91125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Correcting fro data imbalance (only about 1.78% malignant samples in train set)\ntotal_img = train_csv['target'].size\nmalignant = np.count_nonzero(train_csv['target'])\nbenign = total_img - malignant\ninitial_bias = np.log([malignant/benign])\ninitial_bias","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:00:10.267191Z","iopub.execute_input":"2022-03-20T19:00:10.270373Z","iopub.status.idle":"2022-03-20T19:00:10.27919Z","shell.execute_reply.started":"2022-03-20T19:00:10.270334Z","shell.execute_reply":"2022-03-20T19:00:10.278275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nDue to having fewer malignant images, we want these malignant images to have more weight in our model. By increasing the weight of these malignant images, the model will pay more attention to them, and this will help balance out the difference in quantity.","metadata":{}},{"cell_type":"code","source":"weight_for_0 = (1 / benign)*(total_img)/2.0 \nweight_for_1 = (1 / malignant)*(total_img)/2.0\n\nclass_weight = {0: weight_for_0, 1: weight_for_1}\n\nprint('Weight for class 0: {:.2f}'.format(weight_for_0))\nprint('Weight for class 1: {:.2f}'.format(weight_for_1))","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:00:14.632323Z","iopub.execute_input":"2022-03-20T19:00:14.632575Z","iopub.status.idle":"2022-03-20T19:00:14.644434Z","shell.execute_reply.started":"2022-03-20T19:00:14.632547Z","shell.execute_reply":"2022-03-20T19:00:14.643644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = make_model(output_bias = initial_bias, metrics=tf.keras.metrics.AUC(name='auc'))","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:00:31.488511Z","iopub.execute_input":"2022-03-20T19:00:31.48907Z","iopub.status.idle":"2022-03-20T19:00:33.611195Z","shell.execute_reply.started":"2022-03-20T19:00:31.489032Z","shell.execute_reply":"2022-03-20T19:00:33.610493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:00:40.655176Z","iopub.execute_input":"2022-03-20T19:00:40.655458Z","iopub.status.idle":"2022-03-20T19:00:40.671927Z","shell.execute_reply.started":"2022-03-20T19:00:40.655425Z","shell.execute_reply":"2022-03-20T19:00:40.670944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"melanoma_model.h5\",\n                                                    save_best_only=True)\n\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=10,\n                                                     restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:00:46.650205Z","iopub.execute_input":"2022-03-20T19:00:46.650888Z","iopub.status.idle":"2022-03-20T19:00:46.657263Z","shell.execute_reply.started":"2022-03-20T19:00:46.65085Z","shell.execute_reply":"2022-03-20T19:00:46.656511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    train_dataset, epochs=2,\n    steps_per_epoch=STEPS_PER_EPOCH,\n    validation_data=valid_dataset,\n    validation_steps=VALID_STEPS,\n    callbacks=[checkpoint_cb, early_stopping_cb, lr_scheduler],\n    class_weight=class_weight\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:00:56.451042Z","iopub.execute_input":"2022-03-20T19:00:56.451636Z","iopub.status.idle":"2022-03-20T19:19:14.954168Z","shell.execute_reply.started":"2022-03-20T19:00:56.451577Z","shell.execute_reply":"2022-03-20T19:19:14.953403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = get_test_dataset(ordered=True)\n\nprint('Computing predictions...')\ntest_images_ds = test_ds.map(lambda image, idnum: image)\nprobabilities = model.predict(test_images_ds)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:20:16.137484Z","iopub.execute_input":"2022-03-20T19:20:16.137844Z","iopub.status.idle":"2022-03-20T19:22:56.838771Z","shell.execute_reply.started":"2022-03-20T19:20:16.137806Z","shell.execute_reply":"2022-03-20T19:22:56.838016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample prediction file format\nsub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:25:40.231322Z","iopub.execute_input":"2022-03-20T19:25:40.231606Z","iopub.status.idle":"2022-03-20T19:25:40.261377Z","shell.execute_reply.started":"2022-03-20T19:25:40.231572Z","shell.execute_reply":"2022-03-20T19:25:40.260583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Generating submission.csv file...')\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:26:09.159715Z","iopub.execute_input":"2022-03-20T19:26:09.160407Z","iopub.status.idle":"2022-03-20T19:29:31.327312Z","shell.execute_reply.started":"2022-03-20T19:26:09.16037Z","shell.execute_reply":"2022-03-20T19:29:31.326573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(probabilities)})\npred_df.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-03-20T19:32:15.943033Z","iopub.execute_input":"2022-03-20T19:32:15.943293Z","iopub.status.idle":"2022-03-20T19:32:15.971721Z","shell.execute_reply.started":"2022-03-20T19:32:15.943262Z","shell.execute_reply":"2022-03-20T19:32:15.970998Z"},"trusted":true},"execution_count":null,"outputs":[]}]}