{"cells":[{"metadata":{"_uuid":"a6de38c0ff88b6738c5c8751c722c9b489602d88"},"cell_type":"markdown","source":" # Pipeline Resnet\nThanks to https://www.kaggle.com/dimitreoliveira/aptos-blindness-detection-eda-and-keras-resnet50\n\nThe novelty of this kernel is the attention mechanism that I tried to implement after the resnet50. I display the images with their masks generated by the attention network. This is a first attempt, and needs to be explored a bit more. For me it improved the predictions by quite a lot.   "},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport shutil\nfrom tqdm import tqdm\n\nDATA_PATH = \"../input/aptos2019-blindness-detection/\"\nTRAIN_PATH = DATA_PATH + 'train_images/'\nTEST_PATH = DATA_PATH + 'test_images/'\n\nprint(os.listdir(DATA_PATH))\n\nfrom glob import glob \n\n\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d06b98f87cfa19e2548b0d3a558128d563942e1b"},"cell_type":"markdown","source":"# Load data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(DATA_PATH + \"train.csv\")\ntest = pd.read_csv(DATA_PATH + \"test.csv\")\n\ntrain[\"id_code\"] = train[\"id_code\"].apply(lambda x: x + \".png\")\ntest[\"id_code\"] = test[\"id_code\"].apply(lambda x: x + \".png\")\n\ntrain['diagnosis'] = train['diagnosis'].astype('str')\n\ntrain.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Preprocessing the data before training (and using flow_from_directory instead of flow_from_dataframe) cuts training time within kaggle kernel by factor >10 !**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dest = 'base_dir/train'\ntest_dest =  'base_dir/test/'\n\nif os.path.exists(train_dest):\n    shutil.rmtree(train_dest)\nif os.path.exists(test_dest):\n    shutil.rmtree(test_dest)\n    \n# creating train and validation directories and subdirectories\n\nfor subf in [\"0\", \"1\",\"2\",\"3\",\"4\"]:\n    os.makedirs(os.path.join(train_dest, subf))\n\n# Creating test directory\nos.makedirs(test_dest + 'none/')\n        ","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def crop_image1(img,tol=7):\n    # img is image data\n    # tol  is tolerance\n        \n    mask = img>tol\n    return img[np.ix_(mask.any(1),mask.any(0))]\n\n# The above code work only for 1-channel. Here is my simple extension for 3-channels image\ndef crop_image(img,tol=7):\n    if img.ndim ==2:\n        mask = img>tol\n        return img[np.ix_(mask.any(1),mask.any(0))]\n    elif img.ndim==3:\n        h,w,_=img.shape\n#         print(h,w)\n        img1=cv2.resize(crop_image1(img[:,:,0]),(w,h))\n        img2=cv2.resize(crop_image1(img[:,:,1]),(w,h))\n        img3=cv2.resize(crop_image1(img[:,:,2]),(w,h))\n        \n#         print(img1.shape,img2.shape,img3.shape)\n        img[:,:,0]=img1\n        img[:,:,1]=img2\n        img[:,:,2]=img3\n        return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from cv2 import imread, cvtColor, resize, addWeighted, GaussianBlur, COLOR_BGR2GRAY, imwrite\n\nIMAGE_SIZE = 224\n\n\nfor label in ['0', '1', '2', '3', '4']:\n    for image in tqdm(train[train['diagnosis'] == label].values):\n\n        fname = image[0]\n\n        src = os.path.join(TRAIN_PATH, fname) \n        dst = os.path.join(train_dest, label, fname)\n        \n        im = imread(src)\n        im = cvtColor(im, COLOR_BGR2GRAY)\n        \n        im = crop_image1(im)\n        \n        resized_image = resize(im, (IMAGE_SIZE, IMAGE_SIZE))\n        resized_image = addWeighted(resized_image, 4, GaussianBlur(resized_image, (0,0), IMAGE_SIZE/10), -4, 128)\n        \n        status = imwrite(dst, resized_image)\n        \n   \nprint('\\nPreparing test images...')      \nfor image in tqdm(test.values):\n\n        fname = image[0]\n\n        src = os.path.join(TEST_PATH, fname) \n        dst = os.path.join(test_dest  + 'none/', fname)\n        \n        im = imread(src)\n        im = cvtColor(im, COLOR_BGR2GRAY)\n        resized_image = resize(im, (IMAGE_SIZE, IMAGE_SIZE))\n        resized_image = addWeighted(resized_image , 4, GaussianBlur(resized_image, (0,0), IMAGE_SIZE/10), -4, 128)\n        \n        imwrite(dst, resized_image)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Display some preprocessed images**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib.pyplot import figure, imshow\nfrom cv2 import imread\n\nSEED = 101\n\nfig = figure(figsize=(25, 16))\n\nfor class_id in ['0', '1', '2', '3', '4']:\n    for i, (idx, row) in enumerate(train.loc[train['diagnosis'] == class_id].sample(5, random_state=SEED).iterrows()):\n        ax = fig.add_subplot(5, 5, int(class_id) * 5 + i + 1, xticks=[], yticks=[])\n        path= os.path.join(train_dest, class_id, row['id_code'])\n        image = imread(path)\n        #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        #image = cv2.resize(image, (IMG_SIZE, IMG_SIZE))\n\n        imshow(image)\n        ax.set_title('Label: %d-%d-%s' % (int(class_id), idx, row['id_code']) )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0bfaeb9d367e1b720fd81623e9248c3ff7bb0b5e"},"cell_type":"markdown","source":"# Creating the data generators"},{"metadata":{"_uuid":"145ef9177524908eb5656fd1deb55a82aeb01973","trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\nbatch_size_train = 32\nbatch_size_test = 1\n\ntrain_datagen = ImageDataGenerator(rescale=1./255, validation_split=0.1,\n                                   horizontal_flip=True, \n                                   vertical_flip=True,\n                                   rotation_range=20,\n                                   zoom_range= 0.2,\n                                   featurewise_center=True,\n                                   featurewise_std_normalization=True,\n                                   zca_whitening=True,\n                                   width_shift_range=0.2,\n                                   height_shift_range=0.2,)\n\ntrain_gen = train_datagen.flow_from_directory(directory=train_dest,\n                                              batch_size= batch_size_train,\n                                              class_mode= 'categorical',\n                                              target_size=(IMAGE_SIZE, IMAGE_SIZE),\n                                              subset='training')\n\nvalid_gen = train_datagen.flow_from_directory(directory=train_dest,\n                                              batch_size= batch_size_train,\n                                              class_mode= 'categorical',\n                                              target_size=(IMAGE_SIZE, IMAGE_SIZE),\n                                              subset='validation')\n\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntest_gen = test_datagen.flow_from_directory(directory=test_dest,\n                                            batch_size= batch_size_test,\n                                            class_mode= None,\n                                            target_size=(IMAGE_SIZE, IMAGE_SIZE),\n                                            shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"037766b4d6d71232ff280ee04aee45671db93b7a"},"cell_type":"markdown","source":"# Define the model "},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import math, cast, float32\nfrom tensorflow.nn import softmax\nfrom tensorflow.keras.losses import CategoricalCrossentropy\n\ndef custom_loss(y_true, y_pred):\n    \"\"\"\n    Define your code here. You can now use `weights` directly\n    in this function\n    \"\"\"\n    custom_cce = math.add(math.multiply(y_true, -1*math.log(0.001+ y_pred*0.999)), \n                          math.multiply(1-y_true, -1*math.log(1-y_pred*0.999)))\n    custom_cce = math.reduce_sum(custom_cce, 1) \n    \n    weights = cast(1 + math.square(math.abs(math.argmax(y_pred, axis=1)-math.argmax(y_true, axis=1)))/100, float32)\n    \n    \n    cce = CategoricalCrossentropy()\n   \n    return math.multiply(cce(y_pred,y_true), weights)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d252eb588aaf888171ff82b332efd4a0880cab3","trusted":true},"cell_type":"code","source":"from keras.applications import ResNet50\nfrom keras.applications.resnet50 import preprocess_input\nfrom keras import Model, layers\nfrom keras.models import load_model, model_from_json\nfrom keras.layers import GlobalAveragePooling2D, Dropout, Dense, Input\n\nprint('Creating model... ', end=' ')\n\ninput_tensor = Input(shape=(IMAGE_SIZE,IMAGE_SIZE,3))\nconv_base = ResNet50(include_top=False, weights=None, input_tensor=input_tensor)\n\nconv_base.load_weights('../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n\n# Attention network\n\n#a_map = layers.Conv2D(1024, 1, strides=(1, 1), padding=\"same\", activation='relu')(conv_base.output)\n\na_map = layers.Conv2D(516, 1, strides=(1, 1), padding=\"same\", activation='relu')(conv_base.output)\n\n#a_map = layers.Conv2D(64, 1, strides=(1, 1), padding=\"same\", activation='relu')(a_map)\n\na_map = layers.Conv2D(1, 1, strides=(1, 1), padding=\"same\", activation='relu')(a_map)\n\n#a_map = layers.Conv2D(1024, 1, strides=(1, 1), padding=\"same\", activation='relu')(a_map)\n\na_map = layers.Conv2D(2048, 1, strides=(1, 1), padding=\"same\", activation='sigmoid')(a_map)\n\n\n\nres = layers.Multiply()([conv_base.output, a_map])\n    \nx = GlobalAveragePooling2D()(res)\nx = Dropout(0.5)(x)\nx = Dense(2048, activation='relu')(x)\nx = Dropout(0.5)(x)\npredictions = Dense(5, activation='softmax', name='final_output')(x)\nmodel = Model(input_tensor, predictions)\n\nmodel.summary()\n\nprint(\"Done !\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"56a7cce49809eae68d588a542620e16368e4fd1a"},"cell_type":"markdown","source":"# Train"},{"metadata":{},"cell_type":"markdown","source":"**Training upper layers**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom keras.optimizers import Adam\n\ntrain_steps = train_gen.n//train_gen.batch_size\nval_steps = valid_gen.n//valid_gen.batch_size\n\nfor layer in model.layers:\n    layer.trainable = False\n\nfor i in range(-9, 0):\n    model.layers[i].trainable = True\n    \noptimizer = Adam(lr=1e-4)\nmodel.compile(optimizer=optimizer, loss='categorical_crossentropy',  metrics=['accuracy'])\n\nhistory_warmup = model.fit_generator(train_gen,\n                                     steps_per_epoch=train_steps,\n                                     validation_data=valid_gen,\n                                     validation_steps=val_steps,\n                                     epochs=2).history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Training all layers**"},{"metadata":{"_uuid":"a6dc621f8fc4f4558cb22289986a4886fe9e64c8","trusted":true},"cell_type":"code","source":"for layer in model.layers:\n    layer.trainable = True\n\nearlystopper = EarlyStopping(monitor='val_loss', patience=7, verbose=1, restore_best_weights=True)\nreducel = ReduceLROnPlateau(monitor='val_loss', patience=2, verbose=1, factor=0.3, min_lr=1e-6)\n\noptimizer = Adam(1e-4)\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=optimizer,\n              metrics=['accuracy'])\n\n\n\nhistory = model.fit_generator(train_gen,\n                              steps_per_epoch=train_steps,\n                              validation_data=valid_gen,\n                              validation_steps=val_steps,\n                              epochs=25,\n                              callbacks=[reducel, earlystopper]).history","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c8099fc4994ae9792c74477711794c83db3aeee","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\ntraining_data = {'loss': history_warmup['loss'] + history['loss'],\n                 'val_loss': history_warmup['val_loss'] + history['val_loss'],\n                 'acc': history_warmup['acc'] + history['acc'],\n                 'val_acc': history_warmup['val_acc'] + history['val_acc']}\n\nsns.set_style(\"whitegrid\")\nfig, (ax1, ax2) = plt.subplots(2, 1, sharex='col', figsize=(20, 14))\n\nax1.plot(history['loss'], label='Train loss')\nax1.plot(history['val_loss'], label='Validation loss')\nax1.legend(loc='best')\nax1.set_title('Loss')\n\nax2.plot(history['acc'], label='Train Accuracy')\nax2.plot(history['val_acc'], label='Validation accuracy')\nax2.legend(loc='best')\nax2.set_title('Accuracy')\n\nplt.xlabel('Epochs')\nsns.despine()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analysis of the attention mechanism"},{"metadata":{"trusted":true},"cell_type":"code","source":"from cv2 import resize\n\ndef show_image_mask(path):\n\n    im = imread(path)\n\n    exctraction_model = Model(input_tensor, res)\n    im = np.array([im])\n    w = exctraction_model.predict(im)\n    im_res = np.squeeze(im)[:,:,0]\n    \n    # emphasizing the attention for comparaisong purpose\n    im_w = np.power(resize(np.mean(w[0], axis=2), (224, 224)),4)\n    imshow(np.multiply(im_w, im_res), cmap = 'gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = figure(figsize=(25, 16))\n\nfor class_id in ['0', '1', '2', '3', '4']:\n    for i, (idx, row) in enumerate(train.loc[train['diagnosis'] == class_id].sample(5, random_state=SEED).iterrows()):\n        ax = fig.add_subplot(5, 5, int(class_id) * 5 + i + 1, xticks=[], yticks=[])\n        path= os.path.join(train_dest, class_id, row['id_code'])\n        show_image_mask(path)\n        ax.set_title('Label: %d-%d-%s' % (int(class_id), idx, row['id_code']) )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d52bccbbd7d4a8114b5bc9f53ce26db2601fa238"},"cell_type":"markdown","source":"# Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"final_valid_datagen = ImageDataGenerator(rescale=1./255)\n\nfinal_valid_gen = train_datagen.flow_from_directory(directory= train_dest,\n                                              batch_size= 1,\n                                              class_mode= 'categorical',\n                                              target_size= (IMAGE_SIZE, IMAGE_SIZE),\n                                              shuffle=False)\n\nSTEP_SIZE_TEST = final_valid_gen.n//final_valid_gen.batch_size\nprint('Start predictions...', end=' ')\npreds = model.predict_generator(final_valid_gen, steps=STEP_SIZE_TEST)\nprint('Done !')\npredictions = [np.argmax(pred) for pred in preds]\n\ngt= list(final_valid_gen.labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom seaborn import heatmap\n\nplt.figure(figsize=(10, 10))\n\ncm = confusion_matrix(gt, predictions, labels=[0,1,2,3,4])\nheatmap(cm, annot=True, cbar=False, cmap='Blues', fmt='g', vmax=300, vmin=0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ccdefc4770fa3b4d57c56990105173161471420","trusted":true},"cell_type":"code","source":"test_gen.reset()\nSTEP_SIZE_TEST = test_gen.n//test_gen.batch_size\nprint(\"Predictions begins...\")\npreds = model.predict_generator(test_gen, steps=STEP_SIZE_TEST)\nprint(\"Predictions done !\")\npredictions = [np.argmax(pred) for pred in preds]\n\nfilenames = test_gen.filenames\nresults = pd.DataFrame({'id_code':filenames, 'diagnosis':predictions})\nresults['id_code'] = results['id_code'].map(lambda x: str(x)[:-4])\nresults['id_code'] = results['id_code'].apply(lambda x: x.split('/')[1])\nresults.to_csv('submission.csv',index=False)\nresults.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"357d239024be4067fa88d87ca0dc0202a59b9d1b","trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"submission.csv\")\nprint(df[\"diagnosis\"].value_counts())\nprint(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.exists('valid/'):\n    shutil.rmtree('valid/')\nif os.path.exists(train_dest):\n    shutil.rmtree(train_dest)\nif os.path.exists(test_dest):\n    shutil.rmtree(test_dest)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}