{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport math\nimport time\nimport os\n\nimport tensorflow as tf\n\nfrom PIL import Image\n\n# from sklearn.metrics import *\nfrom sklearn.preprocessing import StandardScaler\nfrom tqdm.notebook import tqdm\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as be\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.losses import BinaryCrossentropy\n# from tensorflow_addons.losses import *\nfrom tensorflow.keras.optimizers import Adam\n# from tensorflow.keras.regularizers import *\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import confusion_matrix\n\nimport memory_profiler\nimport gc\nfrom pprint import pprint\nfrom imblearn.over_sampling import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mem_usage(pandas_obj):\n    if isinstance(pandas_obj,pd.DataFrame):\n        usage_b = pandas_obj.memory_usage(deep=True).sum()\n    else: # we assume if not a df it's a series\n        usage_b = pandas_obj.memory_usage(deep=True)\n    usage_mb = usage_b / 1024 ** 2 # convert bytes to megabytes\n    return \"{:03.2f} MB\".format(usage_mb)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"img_path = '/kaggle/input/jpeg-melanoma-256x256/'\nbase_path = '/kaggle/input/siim-isic-melanoma-classification/'\n\ndf_train = pd.read_csv(os.path.join(base_path, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(base_path, 'test.csv'))\ndf_submission = pd.read_csv(os.path.join(base_path, 'sample_submission.csv'))\n\ndef timg(name, test=False):\n    if test:\n        return img_path + 'test/' + name + '.jpg'\n    else:\n        return img_path + 'train/' + name + '.jpg'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_grid(df, cols=9, rows=4):\n    if df.shape[0] == 0:\n        return\n    plt.figure(figsize=(18,9))\n    for i in range(min(df.shape[0], cols * rows)):\n        plt.subplot(rows, cols, i+1, xticks=[], yticks=[])\n        idx = np.random.randint(0, df.shape[0], 1)[0]\n        im = Image.open(timg(df.iloc[idx]['image_name']))\n        plt.imshow(im)\n        plt.xlabel(df.iloc[idx]['benign_malignant'])\n        plt.ylabel(df.iloc[idx]['anatom_site_general_challenge'])\n    plt.show()\n\n# Check young people\nprint('Young folks')\nshow_grid(df_train[(df_train['age_approx'] < 40.0) & (df_train['target'] == 1)])\n# Check diagnosis\nprint('Target 1 with Melanoma')\nshow_grid(df_train[(df_train['diagnosis'] == 'melanoma') & (df_train['target'] == 1)])\n# Check a single patient\n# print(\"Patient IP_0962375\")\n# pat = 'IP_0962375'\n# show_grid(df_train[(df_train['patient_id'] == pat) & (df_train['target'] == 1)])\n# show_grid(df_train[(df_train['patient_id'] == pat) & (df_train['target'] == 0)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Mark male female as 1/0\n# There are only two values, there are some missing values, which should be filled with mode\ndf_train['sex'] = df_train['sex'].replace({ 'female': 0, 'male': 1 })\ndf_test['sex'] = df_test['sex'].replace({ 'female': 0, 'male': 1 })\ndf_train['sex'].fillna(df_train['sex'].mode()[0], inplace=True)\n\n# Remove benign malignant, it's the same as target\ndf_train.drop(['benign_malignant'], inplace=True, axis=1)\n\n# Add dummies for anatom_site_general_challenge\n# Fill the nan's with a new dummy\ndef add_dummies(dataset, column, short_name):\n    dummy = pd.get_dummies(\n        dataset[column], \n        drop_first=True, \n        prefix=short_name, \n        prefix_sep='_',\n        dummy_na=True\n    )\n    merged = pd.concat([dataset, dummy], axis=1)\n    return merged.drop([column], axis=1)\n\ndf_train = add_dummies(df_train, 'anatom_site_general_challenge', 'anatom')\ndf_test = add_dummies(df_test, 'anatom_site_general_challenge', 'anatom')\n\n# Age has some missing values, fill with median\ndf_train['age_approx'].fillna(df_train['age_approx'].median(), inplace=True)\ndf_test['age_approx'].fillna(df_test['age_approx'].median(), inplace=True)\n\n# %% [code]\n# Check how many times are their images taken\ndf_train['image_count'] = df_train['patient_id'].map(df_train.groupby(['patient_id'])['image_name'].count())\ndf_test['image_count'] = df_test['patient_id'].map(df_test.groupby(['patient_id'])['image_name'].count())\n\n# Diagnosis is only in train, removing it\ndf_train.drop(['diagnosis', 'patient_id'], inplace=True, axis=1)\ndf_test.drop(['patient_id'], inplace=True, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['image_name'] = df_train['image_name'].apply(lambda x: timg(x))\ndf_test['image_name'] = df_test['image_name'].apply(lambda x: timg(x, test=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sc = StandardScaler()\n\ndef scale(df, cols_to_remove, fit=False):\n    removed = df[cols_to_remove]\n    df = df.drop(cols_to_remove, axis=1)\n    cols = df.columns\n    if fit:\n        df = sc.fit_transform(df)\n    else:\n        df = sc.transform(df)\n    df = pd.DataFrame(df, columns=cols)\n    df[cols_to_remove] = removed\n    return df\n\ndf_train = scale(df_train, fit=True, cols_to_remove=['image_name', 'target'])\ndf_test = scale(df_test, cols_to_remove=['image_name'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install imbalanced-learn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Over sampling / Under Sampling\n\nX = df_train.drop(['target'], axis=1)\ny = df_train['target']\n\nX_resampled, y_resampled = RandomOverSampler(random_state=0).fit_resample(X, y)\nX_resampled['target'] = y_resampled\n\nprint('Target 1 percentage:',\n    (\n        X_resampled[X_resampled['target'] == 1].shape[0]/\n        X_resampled.shape[0]\n    )*100,\n    '%',\n    'Total:',\n    X_resampled.shape[0]\n)\n\ndf_train = X_resampled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS = 3\n\nX_train = [None] * FOLDS\nX_val = [None] * FOLDS\n\nkfold = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=42)\nsplit = kfold.split(\n    np.arange(len(df_train)),\n    df_train['target'],\n)\nfor fold, (idx_train, idx_val) in enumerate(split):\n    X_train[fold] = df_train.iloc[idx_train]\n    X_val[fold] = df_train.iloc[idx_val]\n    \nX_test = df_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Target 1 percentage:',\n    (\n        df_train[df_train['target'] == 1].shape[0]/\n        df_train.shape[0]\n    )*100,\n    '%'\n)\n\nprint(X_train[0][X_train[0]['target'] == 1].shape[0])\nprint(X_train[1][X_train[1]['target'] == 1].shape[0])\nprint(X_train[2][X_train[2]['target'] == 1].shape[0])\n\nprint(X_val[0][X_val[0]['target'] == 1].shape[0])\nprint(X_val[1][X_val[1]['target'] == 1].shape[0])\nprint(X_val[2][X_val[2]['target'] == 1].shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset():\n    \n    def __init__(self, df, training=True, augment=False, batch_size=16, meta=False):\n        self.auto = tf.data.experimental.AUTOTUNE\n        self.training = training\n        self.augment = augment\n        \n        if meta:\n            if training:\n                self.ds = tf.data.Dataset.from_tensor_slices((\n                    df.drop(['target', 'image_name'], axis=1).values,\n                    df['image_name'].values,\n                    df['target'].values\n                ))\n            else:\n                self.ds = tf.data.Dataset.from_tensor_slices((\n                    df.drop(['image_name'], axis=1).values,\n                    df['image_name'].values,\n                    [-1] * df.shape[0],\n                ))\n            self.ds = self.ds.map(\n                lambda meta, img, target: (meta, self.load_image(img), target), \n                num_parallel_calls=self.auto\n            )\n\n        else:\n            if training:\n                self.ds = tf.data.Dataset.from_tensor_slices((\n                    df['image_name'].values,\n                    df['target'].values\n                ))\n            else:\n                self.ds = tf.data.Dataset.from_tensor_slices((\n                    df['image_name'].values,\n                    [-1] * df.shape[0],\n                ))\n            self.ds = self.ds.map(\n                lambda img, target: (self.load_image(img), target), \n                num_parallel_calls=self.auto\n            )\n\n#         self.ds = self.ds.cache()\n        self.ds = self.ds.batch(batch_size, drop_remainder=True)\n        self.ds = self.ds.prefetch(self.auto)\n\n    def data(self):\n        return self.ds\n    \n    def load_image(self, img):\n        size = 256\n\n        img = tf.io.read_file(img)\n        img = tf.image.decode_jpeg(img, channels=3)\n        img = tf.cast(img, tf.float32) / 255.0\n        \n        if self.augment:\n#             img = tf.keras.layers.experimental.preprocessing.RandomRotation(0.2, fill_mode='reflect')(img)\n#             img = tf.keras.layers.experimental.preprocessing.RandomZoom(0.2)(img)\n            \n            img = tf.image.random_brightness(img, 0.2)\n            img = tf.image.random_contrast(img, 0.8, 1.2)\n            img = tf.image.random_flip_left_right(img)\n            img = tf.image.random_flip_up_down(img)\n            img = tf.image.random_hue(img, 0.2)\n            img = tf.image.random_saturation(img, 0.8, 1.2)\n            img = self.dropout(img)\n    \n        img = tf.reshape(img, [size, size, 3])\n        return img\n\n    def dropout(self, image, DIM=256, PROBABILITY = 0.5, CT = 2, SZ = 0.2):\n        # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n        # output - image with CT squares of side size SZ*DIM removed\n\n        # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n        if (P==0)|(CT==0)|(SZ==0): return image\n\n        for k in range(CT):\n            # CHOOSE RANDOM LOCATION\n            x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n            y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n            # COMPUTE SQUARE \n            WIDTH = tf.cast( SZ*DIM,tf.int32) * P\n            ya = tf.math.maximum(0,y-WIDTH//2)\n            yb = tf.math.minimum(DIM,y+WIDTH//2)\n            xa = tf.math.maximum(0,x-WIDTH//2)\n            xb = tf.math.minimum(DIM,x+WIDTH//2)\n            # DROPOUT IMAGE\n            one = image[ya:yb,0:xa,:]\n            two = tf.zeros([yb-ya,xb-xa,3]) \n            three = image[ya:yb,xb:DIM,:]\n            middle = tf.concat([one,two,three],axis=1)\n            image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n\n        # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n        image = tf.reshape(image,[DIM,DIM,3])\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = Dataset(df_train, augment=True).data()\nprint( '[Train] Total:', train_ds.cardinality().numpy() )\n\ntest_ds = Dataset(df_test, training=False).data()\nprint( '[Test] Total:', test_ds.cardinality().numpy() )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_grid_dataset():\n    sample = train_ds.take(100)\n    count = 0\n    plt.figure(figsize=(18,9))\n    for idx, data in enumerate(iter(sample)):\n        imgs, targets = data\n        if count == 32:\n            break\n        for i, img in enumerate(imgs):\n            plt.subplot(4, 8, count+1, xticks=[], yticks=[])\n            img = np.clip(img.numpy() * 255, 0, 255).astype(np.uint8)\n            img = Image.fromarray(img)\n            plt.xlabel(str(targets[i].numpy()))\n            plt.imshow(img)\n            count += 1\n    plt.show()\n\nif __name__ == '__main__':\n    m1 = memory_profiler.memory_usage()\n    t1 = time.perf_counter()\n    cubes = show_grid_dataset()\n    t2 = time.perf_counter()\n    m2 = memory_profiler.memory_usage()\n    time_diff = t2 - t1\n    mem_diff = m2[0] - m1[0]\n    print(f\"It took {time_diff} Secs and {mem_diff} Mb to execute this method\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications import EfficientNetB4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.backend.clear_session()\ngc.collect()\n\n\ndef get_model():\n    efn0_base = EfficientNetB4(include_top=False, weights='imagenet', input_shape=(256, 256, 3))\n#     efn0_base.trainable = False\n    \n    model = Sequential()\n    model.add(efn0_base)\n    model.add(GlobalAveragePooling2D())\n#     model.add(Dense(32, activation='relu'))\n#     model.add(Dropout(0.4))\n    model.add(Dense(1, activation='sigmoid'))\n    model.summary()\n    \n    model.compile(\n        optimizer=Adam(learning_rate=1e-3),\n        loss=BinaryCrossentropy(label_smoothing=0.05), # \n        metrics=['binary_crossentropy', AUC(name='auc')]\n    )\n    return model\n\nmodel = get_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS=3\nBATCH_SIZE=32\n\nscores = list()\n\ndef train():\n    for i in range(1): #FOLDS\n        fX_train = X_train[i]\n        fX_val = X_val[i]\n\n        train_ds = Dataset(X_train[i], augment=True, batch_size=BATCH_SIZE)\n        val_ds = Dataset(X_val[i], batch_size=BATCH_SIZE)\n        \n        history = model.fit(\n            train_ds.data(),\n            validation_data=val_ds.data(),\n            verbose=1,\n            epochs=EPOCHS,\n            # steps_per_epoch=math.floor(train_ds.data().cardinality()/BATCH_SIZE),\n            batch_size=BATCH_SIZE,\n            callbacks=[\n                # EarlyStopping(monitor='auc', mode='max', patience=6, verbose=2, restore_best_weights=True),\n                ModelCheckpoint(\n                    monitor='val_auc', verbose=1, save_best_only=True, mode='max', filepath='{val_auc:.5f}.h5'),\n                ReduceLROnPlateau(monitor='val_auc', factor=0.1, patience=3, verbose=1, mode='max', cooldown=1, min_lr=0 ),\n            ],\n        )\n        pprint(history.history)\n        \n        predicted = model.predict(train_ds.data(), verbose=1)\n        y_pred = np.reshape(np.round(predicted), (1, predicted.shape[0]))[0]\n        y_true = X_train[i]['target'].iloc[:train_ds.data().cardinality().numpy()*BATCH_SIZE].values\n        print( confusion_matrix(y_true, y_pred) )\n        \ntrain()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get score on the whole data set\ntrain_ds = Dataset(df_train, training=False, batch_size=BATCH_SIZE)\npredicted = model.predict(train_ds.data(), verbose=1)\n\ny_pred = np.reshape(np.round(predicted), (1, predicted.shape[0]))[0]\ny_true = df_train['target'].iloc[:33120].values\n\nconfusion_matrix(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get scores for the test set and submit\nsub_model = tf.keras.models.load_model('./0.91169_efn_pool_batch_256BD10x1.h5') # 0.9074\n\n# Choose sub_model or model\n\ntest_ds = Dataset(X_test, training=False, batch_size=BATCH_SIZE)\npredicted = model.predict(test_ds.data(), verbose=1)\n\nsub = pd.DataFrame(dict(\n    image_name = np.array([img_name.numpy().decode('utf-8') for img, img_name in iter(ds.stream().unbatch()) ]),\n    target = np.reshape(predicted, (1, predicted.shape[0]))[0]\n))\n\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}