{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import glob\nimport numpy as np\nimport pandas as pd\nimport os\nimport PIL\nimport matplotlib.pyplot as plt\nimport math\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OneHotEncoder\n\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img\nfrom keras.models import Sequential, Model\nfrom keras.layers import Conv2D, MaxPooling2D, concatenate\nfrom keras.layers import Activation, Dropout, Flatten, Dense, Input\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.callbacks import Callback","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"cat_dtype = {'sex':'category', 'anatom_site_general_challenge':'category'}\n\ntrain_df = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/train.csv\", dtype=cat_dtype)\ntest_df = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/test.csv\", dtype=cat_dtype)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define the function to change the category column from string to int16\ndef chg_cat_int(df, cat_col):\n    \n    for col, col_dtype in cat_col.items():\n        if col_dtype == 'category':\n            df[col] = df[col].cat.codes.astype('int16')\n            df[col] -= df[col].min()\n            \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fill the NA value and change the category string to int\ntrain_df['sex'] = train_df['sex'].cat.add_categories('unknown').fillna('unknown')\ntrain_df['anatom_site_general_challenge'] = train_df['anatom_site_general_challenge'].cat.add_categories('unknown').fillna('unknown')\ntrain_df['age_approx'] = train_df['age_approx'].fillna(train_df['age_approx'].mean())\n\ntrain_df = chg_cat_int(train_df, cat_dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['sex'] = test_df['sex'].cat.add_categories('unknown').fillna('unknown')\ntest_df['anatom_site_general_challenge'] = test_df['anatom_site_general_challenge'].cat.add_categories('unknown').fillna('unknown')\ntest_df['age_approx'] = test_df['age_approx'].fillna(test_df['age_approx'].mean())\n\ntest_df = chg_cat_int(test_df, cat_dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Training dataset: Number of data with target=1: ', len(train_df[train_df['target'] == 1]))\nprint('Training dataset: Number of data with target=0: ', len(train_df[train_df['target'] == 0]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see there is huge gap between number of the normal samples and the Melanoma samples. If we put both samples directly into model training, it may show good training accuracy but actually it may not correctly identify the Melanoma cases due to small provided samples. Therefore, we actually don't need to load all the normal samples but it would be better to make sure all the Melanoma samples are used for model training. Furthermore, it would be better to find ways to produce more Melanoma samples for training, like rotate the image for 90 degree as a new sample. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/train/'\ntest_path = '/kaggle/input/siim-isic-melanoma-classification/jpeg/test/'\n\ntrain_df['file_path'] = train_path + train_df['image_name'] + '.jpg'\ntest_df['file_path'] = test_path + test_df['image_name'] + '.jpg'\n\ncols = ['image_name', 'patient_id', 'sex', 'age_approx', 'anatom_site_general_challenge', 'diagnosis', 'benign_malignant', 'file_path', 'target']\ntrain_df = train_df[cols]\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"enc = OneHotEncoder(categories=[[0, 1, 2], [0, 1, 2, 3, 4, 5, 6]], handle_unknown='ignore')\nenc_train_df = pd.DataFrame(enc.fit_transform(train_df[['sex', 'anatom_site_general_challenge']]).toarray())\nenc_test_df = pd.DataFrame(enc.fit_transform(test_df[['sex', 'anatom_site_general_challenge']]).toarray())\n\ntrain_df = train_df.join(enc_train_df)\ntest_df = test_df.join(enc_test_df)\n\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Checking the provided images of patient skin, we noticed that the lesion are mainly in the center of the image, the surrounding part of the lesion are mainly skin and hairs. To reduce the noice of the data, to crop the center part of the image would be a good way to improve the accuracy. \n\nSo below we define a function to crop the center part by providing the new width and height value and an example to show the difference between orignal image and cropped image. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# define the function to crop the center part of the image\ndef crop_center(img, cropx, cropy):\n    y,x = img.shape[1:3]\n    startx = x//2-(cropx//2)\n    starty = y//2-(cropy//2)    \n    return img[:,starty:starty+cropy,startx:startx+cropx:,]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_1 = np.array([img_to_array(load_img(test_df['file_path'][0], target_size=(400,400)))])\nimage_1 /= 255.0\ncrop_image_1 = crop_center(image_1, 300, 300)\n\nf, axarr = plt.subplots(1,2, figsize=(8,15)) \naxarr[0].imshow(image_1[0])\naxarr[0].title.set_text(\"Original image\")\naxarr[1].imshow(crop_image_1[0])\naxarr[1].title.set_text(\"Cropped image\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As said before, it only have 584 samples of Melanoma images. But we can produce more samples by rotating the original images. Below is a sample to rotate the image for 90 degree and it will produce a new sample for the Melanoma case. And the new sample target label would be 1 and the meta data will copy the same data from the original image. Through this way, more samples can be produced to improve the model accuracy on the Melanoma case. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_file = train_df['file_path'][train_df['target']==1].iloc[8]\nimages = np.array([img_to_array(load_img(train_file, target_size=(400,400)))])\nimages /= 255.0\nnew_img = np.rot90(images[0])\n\nf, axarr = plt.subplots(1,2, figsize=(8,15)) \naxarr[0].imshow(images[0])\naxarr[0].title.set_text(\"Original image\")\naxarr[1].imshow(new_img)\naxarr[1].title.set_text(\"rotate 90 degree\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split the original training dataset as the training data and validation data\n\nX_train, X_validate, y_train, y_validate = train_test_split(train_df, train_df['target'], test_size=0.2, random_state=9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_cancer_len = len(X_train[X_train['target'] == 1])\nX_validate_cancer_len = len(X_validate[X_validate['target'] == 1])\nprint(\"Number of Melanoma in training set:\", X_train_cancer_len)\nprint(\"Number of Melanoma in validation set:\", X_validate_cancer_len)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Customized Image Generator\nThere are total more than 30k images, it will cost huge memory to load all the images at once, especially with higher resolution loading parameter. At the same time, it has to keep reasonable resolution for the images to make sure the model training can locate the difference of different images. Therefore, using an image generator to provide the training and validation data would be a good solution. \n\nKeras provided build-in ImageDataGenerator to generate the images data by defined batch size. But it also had some limitation if the provided API don't fulfill your model requirement. You can write your own image generator but it cannot support the multi-threading. Keras provided a solution that you can write your own image generator inheritted from the tf.keras.utils.Sequence() class which enable multi-threading support. \n\nBelow we have written our own image generator inheritted from Sequence() class providing following functions:\n- Not select all the normal images but only num_samples and all the Melanoma images for training, it will highly improve the image loading time. \n- Randomly select the normal image samples during the start of each epoch. \n- Crop the center part of the selected images as the output for model training. \n- Produce more training samples of Melanoma images by rotating the original images by 90 degrees and use the same metadata of original images. \n- Output both image data and metadata for Multi-task model training\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Train_Generator(Sequence):\n\n    def __init__(self, train_df, num_samples=5000, batch_size=50, target_dim=(400, 400)):\n        \n        # separate the data in normal and cancer status\n        normal_data = train_df[train_df['target'] == 0]\n        cancer_data = train_df[train_df['target'] == 1]\n\n        # shuffle the normal data and take num_samples records only to save training time\n        normal_data = normal_data.sample(frac=1).reset_index(drop=True)\n        normal_data = normal_data[:num_samples] \n        \n        # append the cancer data and product X and y for model training data and shuffle the sequence\n        normal_data = normal_data.append(cancer_data, ignore_index=True)\n        normal_data = normal_data.sample(frac=1).reset_index(drop=True)\n        x_image_set = normal_data['file_path']\n        y_set = normal_data['target']\n        cols = [i for i in range(10)]\n        cols.append('age_approx')\n        x_meta_set = normal_data[cols]\n            \n        self.df = train_df\n        self.x_image, self.x_meta, self.y = x_image_set, x_meta_set, y_set\n        self.samples = num_samples\n        self.batch_size = batch_size\n        self.target_size = target_dim\n        #print('Init function, self.x=', self.x_image[0], ', lengh=', len(self.x_image))\n\n    def __len__(self):\n        return math.ceil(len(self.x_image) / self.batch_size)\n\n    def __getitem__(self, idx):\n        #print('index:', idx)\n        batch_x = self.x_image[idx * self.batch_size:(idx + 1) * self.batch_size]\n        batch_y = np.array(self.y[idx * self.batch_size:(idx + 1) * self.batch_size])\n        image_x = np.array([img_to_array(load_img(img, target_size=self.target_size)) for img in batch_x])\n        image_x /= 255.0\n        \n        crop_image_x = crop_center(image_x, self.target_size[0]-100, self.target_size[1]-100)\n        meta_x = self.x_meta[idx * self.batch_size:(idx + 1) * self.batch_size]\n        \n        for i in range(len(batch_y)):\n            if batch_y[i] == 1:\n                new_img = np.rot90(crop_image_x[i])\n                crop_image_x = np.concatenate((crop_image_x, [new_img]))\n                batch_y = np.concatenate((batch_y, [1]))\n                new_meta = meta_x.iloc[i]\n                meta_x = meta_x.append(new_meta)      \n        #print('target size:', batch_y)\n\n        return [crop_image_x, meta_x], batch_y\n    \n    def on_epoch_end(self):\n        normal_data = self.df[self.df['target'] == 0]\n        cancer_data = self.df[self.df['target'] == 1]\n        \n        normal_data = normal_data.sample(frac=1).reset_index(drop=True)\n        normal_data = normal_data[:self.samples] \n        \n        normal_data = normal_data.append(cancer_data, ignore_index=True)\n        normal_data = normal_data.sample(frac=1).reset_index(drop=True)\n        x_image_set = normal_data['file_path']\n        y_set = normal_data['target']\n        cols = [i for i in range(10)]\n        cols.append('age_approx')\n        x_meta_set = normal_data[cols]\n        \n        self.x_image, self.x_meta, self.y = x_image_set, x_meta_set, y_set","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Multi-task Learning\nBelow we use the Multi-task Learning concept to setup the model for training and prediction. Multi-task Learning is a subfield of machine learning in which multiple learning tasks are solved at the same time. It can accept different input data with same target or different input data and different target. Here we have 2 different input data, image data and meta data, and same training target. Since the CNN deep learing model are very good at image data training, we can setup a Multi-task Learning model combining a CNN model for image data input and a simple MLP model for meta data input. \n- CNN: Convolutional neural network, CNN is a class of deep neural networks, most commonly applied to analyzing visual imagery.\n- MLP: Multilayer perceptron, MLP is a class of feedforward artificial neural network that composed of multiple layers of perceptrons (with threshold activation). ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# create the CNN model\ndef create_cnn(input_dim):\n\n    model = Sequential()\n\n    # Convolutional layer and max pooling layer\n    model.add(Conv2D(32, (3, 3), activation='relu', input_shape=input_dim))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Conv2D(64, (3, 3), activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    #model.add(Dropout(0.25))\n\n    model.add(Conv2D(128, (3, 3), activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n\n    model.add(Conv2D(128, (3, 3), activation='relu'))\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    #model.add(Dropout(0.25))\n\n    model.add(Flatten())\n    model.add(Dense(2048, activation='relu', name='deep_1'))\n    model.add(Dense(1024, activation='relu', name='deep_2'))\n    model.add(Dense(256, activation='relu'))\n    #model.add(Dropout(0.5))\n    #model.add(Dense(1, activation='sigmoid'))\n\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create the MLP model\ndef create_mlp(input_dim):\n    \n    model = Sequential()\n    model.add(Dense(16, input_dim=input_dim, activation=\"relu\"))\n    model.add(Dense(8, activation=\"relu\"))\n   \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Combine the CNN and MLP model\nmlp = create_mlp(11)\ncnn = create_cnn((300, 300, 3))\n\ncombinedInput = concatenate([mlp.output, cnn.output])\n\n# Adding the classification layer\nx = Dense(64, activation=\"relu\")(combinedInput)\nx = Dense(1, activation=\"sigmoid\")(x)\n\nmodel = Model(inputs=[cnn.input, mlp.input], outputs=x)\n\nmodel.compile(loss='binary_crossentropy',\n                  optimizer=optimizers.RMSprop(lr=1e-4),\n                  metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define the batch size and the image loading resolution\nbatch_size = 50\ntarget_dim = (400, 400)\n\n# train the model with the customized image generator. Enable multi-thread with workers=6 and run for 8 epochs\nhistory = model.fit_generator(\n    Train_Generator(X_train, 5000, batch_size, target_dim),\n    steps_per_epoch = (5000+X_train_cancer_len) // batch_size,\n    validation_data = Train_Generator(X_validate, 800, batch_size, target_dim),\n    validation_steps = (800+X_validate_cancer_len) // batch_size,\n    epochs = 6,\n    workers=6,\n    use_multiprocessing=True,\n    verbose = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test Image Generator and Prediction\nAfter the model is trained, a test data image generator is required to prepare the data for prediction. It's more simple compare to the training image generator. It would only require to load the test images as array format by batch size. And then output the data including metadata prepared for model prediction. \n\nAnd this generator also support multi-thread so that the prediction can save processing time with worker thread as 6. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# The image generator for the test dataset\nclass Test_Generator(Sequence):\n\n    def __init__(self, x_image, x_meta, batch_size=50, target_dim=(200, 200)):\n        self.x_files = x_image\n        self.x_meta = x_meta\n        self.batch_size = batch_size\n        self.target_size = target_dim\n\n    def __len__(self):\n        return math.ceil(len(self.x_files) / self.batch_size)\n\n    def __getitem__(self, idx):\n\n        batch_x = self.x_files[idx * self.batch_size:(idx + 1) * self.batch_size]\n        image_x = np.array([img_to_array(load_img(img, target_size=self.target_size)) for img in batch_x])\n        image_x /= 255.0\n        \n        crop_image_x = crop_center(image_x, self.target_size[0]-100, self.target_size[1]-100)     \n        meta_x = self.x_meta[idx * self.batch_size:(idx + 1) * self.batch_size]\n\n        return [crop_image_x, meta_x]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use the trained model to predict the result\n\ntest_files = test_df['file_path']\ncols = [i for i in range(10)]\ncols.append('age_approx')\nmeta_data = test_df[cols]\n\ntest_predict = model.predict(Test_Generator(test_files, meta_data, batch_size, target_dim),\n                                                   steps=(len(test_files) // batch_size)+1,\n                                                 workers=6, use_multiprocessing=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Output the result for Kaggle submission\n\ntest_df['target'] = test_predict\ntest_df[['image_name', 'target']].to_csv('multitask-submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Summary\n\nFrom the","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}