{"cells":[{"metadata":{},"cell_type":"markdown","source":"### **Thanks to nroman for the advanced hair & microscope view augmentation**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install image_tabular","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision import *\nfrom fastai.tabular import *\nfrom image_tabular.core import *\nfrom image_tabular.dataset import *\nfrom image_tabular.model import *\nfrom image_tabular.metric import *\n\n# use gpu by default if available\n# device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module=\"torch.nn.functional\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = Path(\"../input/resize-jpg-siimisic-melanoma-classification/640x640\")\ndf_path = Path(\"../input/siim-isic-melanoma-classification\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(df_path/\"train.csv\")\ntest_df = pd.read_csv(df_path/\"test.csv\")\n\nprint(len(train_df), len(test_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# extremely unbalanced dataset, most of the images are benign\ntrain_df[\"target\"].value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"size = 256","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cutout_frac = 0.25\np_cutout = 0.75\ncutout_sz = round(size*cutout_frac)\ncutout_tfm = cutout(n_holes=(1,1), length=(cutout_sz, cutout_sz), p=p_cutout)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#hair addition \nimport cv2\nfrom glob import glob\n\nn_max=16     # the maximum number of hairs to augment\nim_size=size  # all images are resized to this size\n\nhair_images=glob('/kaggle/input/melanoma-hairs/*.png')\n\ndef _hair_aug_ocv(input_img):\n    img1 = image2np(input_img)*255 # convert to numpy array in range 0-255\n    img1 = img1.astype(np.uint8) # convert to int\n#     print(img1)\n    \n    img=img1.copy()\n    # Randomly choose the number of hairs to augment (up to n_max)\n    n_hairs = random.randint(0, n_max)\n\n    # If the number of hairs is zero then do nothing\n    if not n_hairs:\n        x = pil2tensor(img, dtype=np.float32)\n        x.div_(255)\n        return x\n\n    # The image height and width (ignore the number of color channels)\n    im_height, im_width, _ = img.shape \n\n    for _ in range(n_hairs):\n\n        # Read a random hair image\n        hair = cv2.imread(random.choice(hair_images)) \n        \n        # Rescale the hair image to the right size (256 -- original size)\n        scale=im_size/256\n        hair = cv2.resize(hair, (int(scale*hair.shape[1]), int(scale*hair.shape[0])), \n                          interpolation=cv2.INTER_AREA)       \n\n        # Flip it\n        # flipcode = 0: flip vertically\n        # flipcode > 0: flip horizontally\n        # flipcode < 0: flip vertically and horizontally    \n        hair = cv2.flip(hair, flipCode=random.choice([-1, 0, 1]))\n\n        # Rotate it\n        hair = cv2.rotate(hair, rotateCode=random.choice([cv2.ROTATE_90_CLOCKWISE,\n                                                          cv2.ROTATE_90_COUNTERCLOCKWISE,\n                                                          cv2.ROTATE_180\n                                                         ])\n                         )\n        \n        \n        # The hair image height and width (ignore the number of color channels)\n        h_height, h_width, _ = hair.shape\n\n        # The top left coord's of the region of interest (roi)  \n        # where the augmentation will be performed\n        roi_h0 = random.randint(0, im_height - h_height)\n        roi_w0 = random.randint(0, im_width - h_width)\n\n        # The region of interest\n        roi = img[roi_h0:(roi_h0 + h_height), roi_w0:(roi_w0 + h_width)]\n\n        # Convert the hair image to grayscale\n        hair2gray = cv2.cvtColor(hair, cv2.COLOR_BGR2GRAY)\n\n        # If the pixel value is smaller than the threshold (10), it is set to 0 (black), \n        # otherwise it is set to a maximum value (255, white).\n        # ret -- the list of thresholds (10 in this case)\n        # mask -- the thresholded image\n        # The original image must be a grayscale image\n        # https://docs.opencv.org/master/d7/d4d/tutorial_py_thresholding.html\n        ret, mask = cv2.threshold(hair2gray, 10, 255, cv2.THRESH_BINARY)\n\n        # Invert the mask\n        mask_inv = cv2.bitwise_not(mask)\n\n        # Bitwise AND won't be performed where mask=0\n        img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)\n        hair_fg = cv2.bitwise_and(hair, hair, mask=mask)\n        # Fixing colors\n        hair_fg = cv2.cvtColor(hair_fg, cv2.COLOR_BGR2RGB)\n        # Overlapping the image with the hair in the region of interest\n        dst = cv2.add(img_bg, hair_fg)\n        # Inserting the result in the original image\n        img[roi_h0:roi_h0 + h_height, roi_w0:roi_w0 + h_width] = dst\n        \n    x = pil2tensor(img, dtype=np.float32)\n    x.div_(255)\n    return x ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# microscope view\np_micro = 0.3\ndef _microscope(input_img):\n    img1 = image2np(input_img)*255 # convert to numpy array in range 0-255\n    img1 = img1.astype(np.uint8) # convert to int\n#     print(img1)\n    \n    img=img1.copy()\n\n    if random.random() < p_micro:\n        circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8),\n                        (img.shape[0]//2, img.shape[1]//2),\n                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15),\n                        (0, 0, 0),\n                        -1)\n\n        mask = circle - 255\n        img = np.multiply(img, mask)\n    x = pil2tensor(img, dtype=np.float32)\n    x.div_(255)\n    return x\nmicroscope = TfmPixel(_microscope)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hair_aug_ocv = TfmPixel(_hair_aug_ocv)\ntfms = get_transforms(flip_vert=True, xtra_tfms = [cutout_tfm, hair_aug_ocv(),microscope()])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# idx for validation, shared by image and tabular data\nval_idx = get_valid_index(train_df)\nlen(val_idx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load image data using train_df and prepare fastai LabelLists\nimage_data = (ImageList.from_df(train_df, path=data_path, cols=\"image_name\",\n                               folder=\"train\", suffix=\".jpg\")\n              .split_by_idx(val_idx)\n              .label_from_df(cols=\"target\")\n              .transform(tfms, size=size))\n\n# add test data so that we can make predictions\ntest_image_data = ImageList.from_df(test_df, path=data_path, cols=\"image_name\",\n                                    folder=\"test\", suffix=\".jpg\")\n\nimage_data.add_test(test_image_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show one example image\n# print(image_data.train[0][1])\nimage_data.train[6][0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tabular data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dep_var = 'target'\ncat_names = ['sex', 'anatom_site_general_challenge']\ncont_names = ['age_approx']\nprocs = [FillMissing, Categorify, Normalize]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tab_data = (TabularList.from_df(train_df, path=data_path, cat_names=cat_names, cont_names=cont_names, procs=procs)\n                           .split_by_idx(val_idx)\n                           .label_from_df(cols=dep_var))\n\n# add test\ntab_data.add_test(TabularList.from_df(test_df, cat_names=cat_names, cont_names=cont_names,\n                                      processor = tab_data.train.x.processor))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# one example\ntab_data.train[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Integrate image and tabular data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"integrate_train, integrate_valid, integrate_test = get_imagetabdatasets(image_data, tab_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# package train, valid, and test datasets into a fastai databunch\nbs = 32\n\ndb = DataBunch.create(integrate_train, integrate_valid, integrate_test,\n                      path=data_path, bs=bs)#.normalize(imagenet_stats)\ndb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# image normalization with imagenet_stats\ndb.norm, db.denorm = normalize_funcs_image_tab(*imagenet_stats)\ndb.add_tfm(db.norm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check the shape of one batch\nx, y = next(iter(db.train_dl))\nlen(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# images\nx[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# categorical and continuous tabular data \nx[1][0].shape, x[1][1].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# targets\ny.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model that trains on image and tabular data simultaneously","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# cnn model for images, use Resnet50 as an example\ncnn_arch = models.resnet50\n\n# cnn_out_sz is the output size of the cnn model that will be concatenated with tabular model output\ncnn_out_sz = 256\n\n# use fastai functions to get a cnn model\nimage_data_db = image_data.databunch()\nimage_data_db.c = cnn_out_sz\ncnn_learn = cnn_learner(image_data_db, cnn_arch, ps=0.2)\ncnn_model = cnn_learn.model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get embedding sizes of categorical data\nemb_szs = tab_data.train.get_emb_szs()\n\n# output size of the tabular model that will be concatenated with cnn model output\ntab_out_sz = 12\n\n# use fastai functions to get a tabular model\ntabular_model = TabularModel(emb_szs, len(cont_names), out_sz=tab_out_sz, layers=[12], ps=0.1)\ntabular_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get an integrated model that combines the two components and concatenate their outputs\n# which will pass through additional fully connected layers\nintegrate_model = CNNTabularModel(cnn_model,\n                                  tabular_model,\n                                  layers = [cnn_out_sz + tab_out_sz, 32],\n                                  ps=0.2,\n                                  out_sz=2).cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check model output dimension, should be (bs, 2)\nintegrate_model(*x).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# adjust loss function weight because the dataset is extremely unbalanced\nweights = [1/(1-train_df[\"target\"].mean()), 1/train_df[\"target\"].mean()]\nloss_func = CrossEntropyFlat(weight=torch.FloatTensor(weights).cuda())#.mixup()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# package everything in a fastai learner, add auc roc score as a metric\nlearn = Learner(db, integrate_model, metrics=[accuracy, ROCAUC()], loss_func=loss_func)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# organize layer groups in order to use differential learning rates provided by fastai\n# the first two layer groups are earlier layers of resnet\n# the last layer group consists of the fully connected layers of cnn model, tabular model,\n# and final fully connected layers for the concatenated data\nlearn.layer_groups = [nn.Sequential(*flatten_model(cnn_learn.layer_groups[0][0])),\n                      nn.Sequential(*flatten_model(cnn_learn.layer_groups[0][1])),\n                      nn.Sequential(*(flatten_model(cnn_learn.layer_groups[0][2]) +\n                                      flatten_model(integrate_model.tabular_model) +\n                                      flatten_model(integrate_model.layers)))]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# find learning rate to train the last layer group first \nlearn.model_dir='/kaggle/working/'\nlearn.model.cuda()\nlearn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train\nlearn.fit_one_cycle(4, 1e-2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# unfreeze all layer groups to train the entire model using differential learning rates\nlearn.unfreeze()\nlearn.fit_one_cycle(8, slice(1e-6, 1e-4))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"val_preds, val_labels = learn.get_preds(DatasetType.Test)\nprint_metrics(val_preds, val_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# # make predictions for the test set\n# preds, y = learn.get_preds(DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# submit predictions to kaggle\nsubmit = pd.read_csv(data_path/\"sample_submission.csv\")\nsubmit[\"target\"] = preds[:, 1]\nsubmit.to_csv(\"/kaggle/working/image_tab.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}