{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reference \n* https://machinelearningmastery.com/how-to-develop-a-generative-adversarial-network-for-a-cifar-10-small-object-photographs-from-scratch/\n* https://www.pyimagesearch.com/2020/03/02/anomaly-detection-with-keras-tensorflow-and-deep-learning/\n","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\nimport os\nimport re\nimport seaborn as sns\nimport numpy as np\nimport pandas as pd\nimport math\nfrom numpy import expand_dims\nfrom numpy import ones\nfrom numpy import zeros\nfrom numpy.random import rand\nfrom numpy.random import randint\nfrom matplotlib import pyplot as plt\n\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\nimport tensorflow_addons as tfa\nfrom sklearn.model_selection import StratifiedKFold\nfrom tqdm import tqdm\n\n#import efficientnet.tfkeras as efn\n\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras import backend as K\nimport tensorflow_addons as tfa\nfrom numpy.random import randn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The inputs are 28x28 RGB images with `channels_last` and the batch  \n# size is 4.  \ninput_shape = (4, 384, 384, 3)\nx = tf.random.normal(input_shape)\ny = tf.keras.layers.Conv2D(3, (5,5), activation='relu', input_shape=input_shape[1:])(x)\nprint(y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"y = tf.keras.layers.Conv2D(3, 3, activation='relu', input_shape=input_shape[1:])(y)\nprint(y.shape)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tflayer = tf.keras.layers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def define_vgg16_encoder(in_shape=(384,384,3)):\n    # Relu modified to LeakyRelu \n    # as described in paper works better for GAN discriminator\n    # using VGG16 as backbone for this\n    with strategy.scope():\n        model = tf.keras.Sequential(name='encoder')\n\n        model.add(tflayer.Conv2D(input_shape=in_shape,filters=64,kernel_size=(3,3),padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.Conv2D(filters=64,kernel_size=(3,3),padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.MaxPool2D(pool_size=(2,2),strides=(2,2)))\n\n        model.add(tflayer.Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.Conv2D(filters=128, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.MaxPool2D(pool_size=(2,2),strides=(2,2)))\n\n        model.add(tflayer.Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.Conv2D(filters=256, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.MaxPool2D(pool_size=(2,2),strides=(2,2)))\n\n        model.add(tflayer.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.MaxPool2D(pool_size=(2,2),strides=(2,2)))\n\n        model.add(tflayer.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.MaxPool2D(pool_size=(2,2),strides=(2,2)))\n    \n        #This is extra layer----- \n        model.add(tflayer.Conv2D(filters=512, kernel_size=(3,3), padding=\"same\", activation=tflayer.LeakyReLU(0.2)))\n        model.add(tflayer.MaxPool2D(pool_size=(2,2),strides=(2,2)))\n        # ------------------------\n        #volumeSize = K.int_shape(model)\n    \n        model.add(tflayer.Flatten())\n\n        model.add(tflayer.Dense(4096, activation=tflayer.LeakyReLU(0.2)))\n        #model.add(tflayer.Dense(1, activation='sigmoid'))\n        # compile model\n        #opt = tf.keras.optimizers.Adam(lr=0.0002, beta_1=0.5)\n        #model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])\n\n        return model\n        #model.add(tflayer.Dense(units=4096,activation=\"relu\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder_model = define_vgg16_encoder((384,384,3))\nencoder_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define the standalone generator model\ndef define_decoder(latent_dim):\n    \n    with strategy.scope():\n        \n        \n        model = tf.keras.Sequential(name='decoder')\n        # same size as just above the falt layer of discriminator\n        n_nodes = 512 * 6 * 6\n        model.add(tflayer.Dense(n_nodes, input_dim=latent_dim))\n        model.add(tflayer.LeakyReLU(alpha=0.2))\n    \n        model.add(tflayer.Reshape((6, 6, 512)))\n        # upsample \n        model.add(tflayer.Conv2DTranspose(512, (4,4), strides=(2,2), padding='same'))\n        model.add(tflayer.LeakyReLU(alpha=0.2))\n\n        # upsample \n        model.add(tflayer.Conv2DTranspose(512, (4,4), strides=(2,2), padding='same'))\n        model.add(tflayer.LeakyReLU(alpha=0.2))\n\n        # upsample \n        model.add(tflayer.Conv2DTranspose(512, (4,4), strides=(2,2), padding='same'))\n        model.add(tflayer.LeakyReLU(alpha=0.2))\n    \n        # upsample \n        model.add(tflayer.Conv2DTranspose(256, (4,4), strides=(2,2), padding='same'))\n        model.add(tflayer.LeakyReLU(alpha=0.2))\n    \n        # upsample \n        model.add(tflayer.Conv2DTranspose(128, (4,4), strides=(2,2), padding='same'))\n        model.add(tflayer.LeakyReLU(alpha=0.2))\n    \n        # upsample \n        model.add(tflayer.Conv2DTranspose(64, (4,4), strides=(2,2), padding='same'))\n        model.add(tflayer.LeakyReLU(alpha=0.2))\n    \n        # output layer\n        model.add(tflayer.Conv2D(3, (3,3), activation='sigmoid', padding='same'))\n        return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Get Decoder","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"latent_dim = 4096\ndecoder_model = define_decoder(latent_dim)\ndecoder_model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Autoencoder","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    \n    inputShape = (384, 384, 3)\n    inputs = tf.keras.Input(shape=inputShape)\n\n    autoencoder = tf.keras.Model(inputs, decoder_model(encoder_model(inputs)),name=\"autoencoder\")\n\n    opt = tfa.optimizers.RectifiedAdam(lr=0.0003)\n    autoencoder.compile(loss=\"mse\", optimizer=opt)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"faeture_list = ['image_name','target','tfrecord']\n\nsiim20_csv = pd.read_csv('../input/jpeg-melanoma-384x384/train.csv',usecols=faeture_list)\nsiim19_csv = pd.read_csv('../input/jpeg-isic2019-384x384/train.csv',usecols=faeture_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"siim19_csv['year'] = '2019' \nsiim20_csv['year'] = '2020'\n\nsiim_all = pd.concat([siim19_csv,siim20_csv],ignore_index = True)\n\ntrain = siim_all.loc[siim_all.target == 1]\nprint('Number of Class 1 images ')\nprint(train.target.value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# REMOVE duplicate images\nfilter_train = train[train.tfrecord != -1 ]\n\nidx_list = []\nfor img_name in filter_train.image_name.values:\n    if img_name.endswith('downsampled'):\n        idx = filter_train.index[filter_train['image_name'] == img_name].to_list()\n        #print(str(idx) + str(len(idx)) + ':' +img_name )\n        if len(idx) == 1:\n            idx_list.append(idx[0])\n\nprint(len(idx_list))\nfilter_train = filter_train.drop(idx_list)\n# shuffle the rows\nfilter_train.reset_index(inplace=True)\n\nfilter_train.drop('index',axis=1)\n\nprint(filter_train.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Taking only 2020 images\nfilter_train = siim20_csv\nfilter_train.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Data access\nGCS_PATH_19 = KaggleDatasets().get_gcs_path('jpeg-isic2019-384x384')\nGCS_PATH_20 = KaggleDatasets().get_gcs_path('jpeg-melanoma-384x384')\n\n#\nSEED_VALUE = 3435\n\n# Configuration\nEPOCHS = 5\nBATCH_SIZE = 4 * strategy.num_replicas_in_sync\nimg_size = 384\nIMAGE_SIZE = [img_size,img_size]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_gcs_path(image_id):\n    \n    year_nb = filter_train.loc[filter_train.image_name == image_id].year.to_numpy()[0]\n    #print(year_nb)\n    GCS_PATH = ''\n    \n    if year_nb == '2019':\n        GCS_PATH = GCS_PATH_19 + '/train/' + image_id + '.jpg'\n    else:\n        GCS_PATH = GCS_PATH_20 + '/train/' + image_id + '.jpg'\n    \n    return GCS_PATH\n\ndef file_path(image_id):\n    \n    year_nb = filter_train.loc[filter_train.image_name == image_id].year.to_numpy()[0]\n    #print(year_nb)\n    GCS_PATH = ''\n    \n    if year_nb == '2019':\n        #print('19')\n        GCS_PATH = '../input/jpeg-isic2019-384x384' + '/train/' + image_id + '.jpg'\n    else:\n        #print('20')\n        GCS_PATH = '../input/jpeg-melanoma-384x384' + '/train/' + image_id + '.jpg'\n    \n    return GCS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filter_train[\"image_path\"] = filter_train[\"image_name\"].apply(lambda x : add_gcs_path(x))\n#filter_train[\"image_jpg_id\"] = filter_train[\"image_name\"].apply(lambda x: file_path(x))\n\nprint(filter_train.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# shuffle the rows\nfilter_train = filter_train.sample(frac=1).reset_index(drop=True)\n\nxtrain, xval, ytrain, yval = train_test_split(filter_train[\"image_path\"], filter_train[\"target\"], \n                                              test_size = 0.10, stratify = filter_train[\"target\"],\n                                              random_state=SEED_VALUE)\n\ndf_train = pd.DataFrame({\"image_path\":xtrain, \"target\":ytrain})\ndf_val = pd.DataFrame({\"image_path\":xval, \"target\":yval})\n\ndf_train[\"target\"] = df_train[\"target\"].astype('int')\ndf_val[\"target\"] = df_val[\"target\"].astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_paths = df_train.image_path.values\nval_paths   = df_val.image_path.values\n\ntrain_labels = df_train.target\nval_labels   = df_val.target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n    \ndef decode_image(filename, label=None, image_size=(img_size, img_size)):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32)\n    # scaling to [-1,1]\n    image = image / 255.0\n    image = tf.image.resize(image, size = image_size)\n    \n    if label is None:\n        return image\n    else:\n        return image, image #label\n\ndef int_div_round_up(a, b):\n    return (a + b - 1) // b","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generate real sample","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((train_paths, train_labels))\n    .map(decode_image, num_parallel_calls=AUTO)\n    #.map(data_augment, num_parallel_calls=AUTO)\n    #.map(transform, num_parallel_calls = AUTO)\n    .repeat()\n    .shuffle(512)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((val_paths, val_labels))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO))\n\nNUM_TRAINING_IMAGES = df_train.shape[0]\nNUM_VALIDATION_IMAGES = df_val.shape[0]\n\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALIDATION_STEPS = int_div_round_up(NUM_VALIDATION_IMAGES, BATCH_SIZE)\nprint('Dataset: {} training images, {} validation images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history1 = autoencoder.fit(\n    train_dataset, \n    validation_data = valid_dataset,\n    validation_steps = VALIDATION_STEPS,\n    epochs=EPOCHS,\n    steps_per_epoch = STEPS_PER_EPOCH\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#acc = history1.history['mse']\n#val_acc = history1.history['val_mse']\n\nloss = history1.history['loss']\nval_loss = history1.history['val_loss']\n\nepochs = range(len(loss))\n\n#plt.plot(epochs, acc, 'b', label='Training mse')\n#plt.plot(epochs, val_acc, 'r', label='Validation mse')\n#plt.title('Training and validation accuracy')\n#plt.legend()\n \nplt.figure()\n \nplt.plot(epochs, loss, 'b', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"autoencoder.save('siim_autoencoder_v1.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Anamoly Detection","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"import cv2\n\ntest_csv = pd.read_csv('../input/jpeg-melanoma-384x384/test.csv')\nimg_dir = '../input/jpeg-melanoma-384x384/test/'\nerrors = []\n\nfor img_name in test_csv.image_name.values:\n    \n    image = cv2.imread(img_dir + img_name + '.jpg')\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) \n    image = image / 255.0\n    #print(image)\n    image = image.reshape((1, 384, 384, 3))\n    \n    decoded_img = autoencoder.predict(image)\n    # Mean of \n    mse = np.mean((image - decoded_img) ** 2)\n    errors.append(decoded_img)\n","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}