{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\nimport seaborn as sns\n\n\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import Dense, Conv2D, SeparableConv2D, MaxPooling2D, GlobalAveragePooling2D, \\\nBatchNormalization, Dropout, Input, Flatten, concatenate\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.callbacks import LearningRateScheduler\n\n# from keras_preprocessing.image import ImageDataGenerator\n\n!pip install -q efficientnet\n\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"WORKING_PATH = \"/kaggle/input/siim-isic-melanoma-classification\"\nTRAIN_DIR = WORKING_PATH + \"/jpeg/train/\"\nTEST_DIR = WORKING_PATH + \"/jpeg/test/\"\nTRAIN_DF_PATH = WORKING_PATH + \"/train.csv\"\nTEST_DF_PATH = WORKING_PATH + \"/test.csv\"\n\n\nBATCH_SIZE = 32\nEPOCHS = 3\nIMAGE_SIZE = (256, 256)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data preprocessing\n\nWe perform the following cleaning and processing to the data: \n\n**Imputation** \n\nFrom a quick EDA that I did, there are some missing values. I decided to fill these in with the modes of each respective field.\n\n**Standard Scaling**\n\nThe age_approx variable is scaled. \n\n**One hot Encoding**\n\nThe categorical variables sex and anatom_site_general_challenge are one-hot encoded. \n\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_DF_PATH)\n\nSS = StandardScaler()\nSS.fit(df_train[[\"age_approx\"]])\n\ndef preprocess_data(df, training=True):\n    # Imputation\n    df = df.fillna({\"sex\": \"male\",\n                    \"age_approx\": df_train[\"age_approx\"].mode().iloc[0],\n                    \"anatom_site_general_challenge\": \"torso\"})\n\n    # get file path \n    DIR = TRAIN_DIR if training else TEST_DIR\n    df[\"file_path\"] = df[\"image_name\"].apply(lambda x : DIR + x + \".jpg\")\n\n    # Scale the age variable\n    df[\"age_approx\"] = SS.transform(df[[\"age_approx\"]])\n\n    # One hot encoding for sex and anatom_site_general_challenge\n    df = pd.get_dummies(df, columns=[\"sex\", \"anatom_site_general_challenge\"])\n    \n    return df\n    \nFEATURE_COLUMNS = ['sex_female',\n 'sex_male',\n 'anatom_site_general_challenge_head/neck',\n 'anatom_site_general_challenge_lower extremity',\n 'anatom_site_general_challenge_oral/genital',\n 'anatom_site_general_challenge_palms/soles',\n 'anatom_site_general_challenge_torso',\n 'anatom_site_general_challenge_upper extremity',\n 'age_approx']\n\ndf_train = preprocess_data(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(TEST_DF_PATH)\n\ndf_test = preprocess_data(df_test, training=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TF datasets\n\nThis is my first experience with these, so I'm still learning. The code below is from [this kernel](https://www.kaggle.com/reighns/groupkfold-efficientbnet-and-augmentations/#Splitting-the-dataset-according-to-GroupKFold). It's a great kernel, so make sure to check it out.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(file_path, label=None):\n    bits = tf.io.read_file(file_path)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, size = IMAGE_SIZE)\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n    \ndef data_augment(image, label=None):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_contrast(image, lower = 1, upper = 2)\n    \n    if label is None:\n        return image\n    else:\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_image_dataset = (tf.data.Dataset.from_tensor_slices(\n    (df_train[\"file_path\"], df_train[FEATURE_COLUMNS]))\n    .map(decode_image, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n    .map(data_augment, num_parallel_calls=tf.data.experimental.AUTOTUNE))\n\n\ntraining_labels_dataset = (tf.data.Dataset.from_tensor_slices(df_train[\"target\"]))\n\ntraining_dataset = (tf.data.Dataset.zip((training_image_dataset, training_labels_dataset))\n           .repeat()\n           .shuffle(512)\n           .batch(BATCH_SIZE)\n           .prefetch(tf.data.experimental.AUTOTUNE)\n          )\n\n\ntesting_dataset = (tf.data.Dataset.from_tensor_slices(\n    (df_test[\"file_path\"], df_test[FEATURE_COLUMNS]))\n    .map(decode_image, num_parallel_calls=tf.data.experimental.AUTOTUNE))\n\ntesting_dataset = (tf.data.Dataset.zip((testing_dataset, )).batch(BATCH_SIZE))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Learning Rate","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lrfn(lr_start=0.00001, lr_max=0.00005, \n               lr_min=0.00001, lr_rampup_epochs=5, \n               lr_sustain_epochs=0, lr_exp_decay=.8):\n    \n    def lrfn(epoch):\n        if epoch < lr_rampup_epochs:\n            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min\n        return lr\n    \n    return lrfn\n\nlrfn = build_lrfn()\nlr_schedule = LearningRateScheduler(lrfn, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Checkpoint","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model_checkpoint = tf.keras.callbacks.ModelCheckpoint('EffNet.h5', save_best_only=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n    \n    # Build the CNN part of the model \n    base_model = efn.EfficientNetB4(weights=\"imagenet\", include_top=False, input_shape=(*IMAGE_SIZE, 3))\n    \n    # Freeze layers \n    for layer in base_model.layers:\n        layer.trainable = False\n    \n    # output -> GlobalPool -> Dense\n    \n    GAP1 = GlobalAveragePooling2D()(base_model.output)\n    CNN_D1 = Dense(400, activation=\"relu\", kernel_initializer=\"he_normal\")(GAP1)\n    CNN_BN1 = BatchNormalization()(CNN_D1)\n    # Add dropout here if overfitting\n    CNN_OUT = Dense(400, activation=\"relu\")(CNN_BN1)\n    \n    # Build the DNN part of the model \n    \n    input_dense = Input((len(FEATURE_COLUMNS), ))\n    DNN_D1 = Dense(400, activation=\"elu\", kernel_initializer=\"he_normal\")(input_dense)\n    DNN_BN1 = BatchNormalization()(DNN_D1)\n    DNN_DO1 = Dropout(0.2)(DNN_BN1)\n    DNN_D2 = Dense(400, activation=\"elu\", kernel_initializer=\"he_normal\")(DNN_D1)\n    DNN_BN2 = BatchNormalization()(DNN_D2)\n    DNN_DO2 = Dropout(0.2)(DNN_BN2)\n    DNN_OUT = Dense(400, activation=\"relu\")(DNN_DO2)\n    \n    # Combine the two \n    \n    concat = concatenate([CNN_OUT, DNN_OUT])\n    output = Dense(1, activation=\"sigmoid\")(concat)\n    \n    model =  Model(inputs=[base_model.input, input_dense], outputs=[output])\n    model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[AUC()])\n    return model \n    \n    \nmodel = build_model()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist = model.fit(training_dataset, epochs=EPOCHS, steps_per_epoch=df_train.shape[0] // BATCH_SIZE, \n                 callbacks = [model_checkpoint, lr_schedule], verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict(testing_dataset, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub = pd.read_csv(WORKING_PATH + \"/sample_submission.csv\")\ndf_sub[\"target\"] = predictions\n\ndf_sub.head()\n\ndf_sub.to_csv(\"submission.csv\", index=False)\n\nmodel.save(\"EffNet.h5\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}