{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import json\nimport math\nimport os\n\nimport cv2\nfrom PIL import Image\nimport numpy as np\nfrom keras import layers\nfrom keras.applications import DenseNet121\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\nimport scipy\nfrom skimage.color import rgb2gray\nimport tensorflow as tf\nfrom tqdm import tqdm\nfrom scipy import ndimage, misc\n\n%matplotlib inline","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-09-21T21:02:14.005559Z","iopub.execute_input":"2022-09-21T21:02:14.005821Z","iopub.status.idle":"2022-09-21T21:02:18.234998Z","shell.execute_reply.started":"2022-09-21T21:02:14.005771Z","shell.execute_reply":"2022-09-21T21:02:18.234141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Set random seed for reproducibility.","metadata":{}},{"cell_type":"code","source":"np.random.seed(2019)\ntf.set_random_seed(2019)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:02:18.236812Z","iopub.execute_input":"2022-09-21T21:02:18.237129Z","iopub.status.idle":"2022-09-21T21:02:18.269804Z","shell.execute_reply.started":"2022-09-21T21:02:18.237055Z","shell.execute_reply":"2022-09-21T21:02:18.269043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nprint(train_df.shape)\nprint(test_df.shape)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:02:18.272842Z","iopub.execute_input":"2022-09-21T21:02:18.273075Z","iopub.status.idle":"2022-09-21T21:02:18.312626Z","shell.execute_reply.started":"2022-09-21T21:02:18.273028Z","shell.execute_reply":"2022-09-21T21:02:18.311828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['diagnosis'].hist()\ntrain_df['diagnosis'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:02:18.313883Z","iopub.execute_input":"2022-09-21T21:02:18.314350Z","iopub.status.idle":"2022-09-21T21:02:18.505407Z","shell.execute_reply.started":"2022-09-21T21:02:18.314298Z","shell.execute_reply":"2022-09-21T21:02:18.504576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Displaying some Sample Images","metadata":{}},{"cell_type":"markdown","source":"We will resize the images to 224x224","metadata":{}},{"cell_type":"code","source":"def unsharp_mask(image, kernel_size=(5, 5), sigma=1.0, amount=1.5, threshold=0):\n    \"\"\"Return a sharpened version of the image, using an unsharp mask.\"\"\"\n    blurred = cv2.GaussianBlur(image, kernel_size, sigma)\n    sharpened = float(amount + 1) * image - float(amount) * blurred\n    sharpened = np.maximum(sharpened, np.zeros(sharpened.shape))\n    sharpened = np.minimum(sharpened, 255 * np.ones(sharpened.shape))\n    sharpened = sharpened.round().astype(np.uint8)\n    if threshold > 0:\n        low_contrast_mask = np.absolute(image - blurred) < threshold\n        np.copyto(sharpened, image, where=low_contrast_mask)\n    return sharpened\n\ndef CLAHEgreen(image):\n    green=image[:, :, 1]\n    clipLimit = 2.0\n    tileGridSize = (8,8)\n    clahe=cv2.createCLAHE(clipLimit = clipLimit, tileGridSize = tileGridSize)\n    cla=clahe.apply(green)\n    img=cv2.merge((cla,cla,cla))\n    \n    return img\n\ndef preprocess_image(path):\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    #image = cv2.resize(image, (224, 224))\n    image=CLAHEgreen(image)\n    image_sharp=unsharp_mask(image)\n    return image_sharp","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:02:18.509329Z","iopub.execute_input":"2022-09-21T21:02:18.509568Z","iopub.status.idle":"2022-09-21T21:02:18.519565Z","shell.execute_reply.started":"2022-09-21T21:02:18.509520Z","shell.execute_reply":"2022-09-21T21:02:18.518358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_image2(path):\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    #image = cv2.resize(image, (224, 224))\n    #image=CLAHEgreen(image)\n    #image_sharp=unsharp_mask(image)\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:02:18.522945Z","iopub.execute_input":"2022-09-21T21:02:18.523325Z","iopub.status.idle":"2022-09-21T21:02:18.533551Z","shell.execute_reply.started":"2022-09-21T21:02:18.523221Z","shell.execute_reply":"2022-09-21T21:02:18.532934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_image1(path):\n    img = cv2.imread(path)\n    #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    #img = cv2.resize(img, (224,224))\n    #image=CLAHEgreen(img)\n    #image_sharp=unsharp_mask(image)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    result1 = ndimage.gaussian_laplace(img, sigma=3)\n    result2=zero_crossing(result1)\n    result3 = cv2.merge((result2,result2,result2))\n    return result3\ndef any_neighbor_zero(img, i, j):\n    for k in range(-1,2):\n        for l in range(-1,2):\n            if img[i+k, j+k] == 0:\n                return True\n    return False\n\ndef zero_crossing(img):\n    img[img > 0] = 1\n    img[img < 0] = 0\n    out_img = np.zeros(img.shape)\n    for i in range(1,img.shape[0]-1):\n        for j in range(1,img.shape[1]-1):\n            if img[i,j] > 0 and any_neighbor_zero(img, i, j):\n                out_img[i,j] = 255\n    return out_img","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:02:18.536100Z","iopub.execute_input":"2022-09-21T21:02:18.536654Z","iopub.status.idle":"2022-09-21T21:02:18.546890Z","shell.execute_reply.started":"2022-09-21T21:02:18.536598Z","shell.execute_reply":"2022-09-21T21:02:18.546294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = train_df.shape[0]\nx_train1 = np.empty((N, 224, 224, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate(tqdm(train_df['id_code'])):\n    x_train1[i, :, :, :] = preprocess_image(\n        f'../input/augmented-clahe/cropped_augmentedclahe_pre/0.9/{image_id}.jpg'\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:02:18.548629Z","iopub.execute_input":"2022-09-21T21:02:18.549099Z","iopub.status.idle":"2022-09-21T21:03:06.404229Z","shell.execute_reply.started":"2022-09-21T21:02:18.548867Z","shell.execute_reply":"2022-09-21T21:03:06.403239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = train_df.shape[0]\nx_train2 = np.empty((N, 224, 224, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate(tqdm(train_df['id_code'])):\n    x_train2[i, :, :, :] = preprocess_image(\n        f'../input/augmented-clahe/cropped_augmentedclahe/0.9/{image_id}.jpg'\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:03:06.405660Z","iopub.execute_input":"2022-09-21T21:03:06.406300Z","iopub.status.idle":"2022-09-21T21:03:56.717375Z","shell.execute_reply.started":"2022-09-21T21:03:06.406003Z","shell.execute_reply":"2022-09-21T21:03:56.716542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = train_df.shape[0]\nx_train3 = np.empty((N, 224, 224, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate(tqdm(train_df['id_code'])):\n    x_train3[i, :, :, :] = preprocess_image2(\n        f'../input/augmented-clahe/cropped_augmentedclahe_pre/0.9/{image_id}.jpg'\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:03:56.718846Z","iopub.execute_input":"2022-09-21T21:03:56.719364Z","iopub.status.idle":"2022-09-21T21:04:05.244176Z","shell.execute_reply.started":"2022-09-21T21:03:56.719140Z","shell.execute_reply":"2022-09-21T21:04:05.243393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = train_df.shape[0]\nx_train4 = np.empty((N, 224, 224, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate(tqdm(train_df['id_code'])):\n    x_train4[i, :, :, :] = preprocess_image2(\n        f'../input/augmented-clahe/cropped_augmentedclahe/0.9/{image_id}.jpg'\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:05.245515Z","iopub.execute_input":"2022-09-21T21:04:05.245985Z","iopub.status.idle":"2022-09-21T21:04:14.365032Z","shell.execute_reply.started":"2022-09-21T21:04:05.245773Z","shell.execute_reply":"2022-09-21T21:04:14.364232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train1 = pd.get_dummies(train_df['diagnosis']).values\ny_train2 = pd.get_dummies(train_df['diagnosis']).values\ny_train3 = pd.get_dummies(train_df['diagnosis']).values\ny_train4 = pd.get_dummies(train_df['diagnosis']).values","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:14.366426Z","iopub.execute_input":"2022-09-21T21:04:14.366875Z","iopub.status.idle":"2022-09-21T21:04:14.378303Z","shell.execute_reply.started":"2022-09-21T21:04:14.366692Z","shell.execute_reply":"2022-09-21T21:04:14.377711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train=np.concatenate((x_train1, x_train2, x_train3, x_train4), axis=0)\ny_train=np.concatenate((y_train1, y_train2, y_train3, y_train4), axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:14.380919Z","iopub.execute_input":"2022-09-21T21:04:14.381229Z","iopub.status.idle":"2022-09-21T21:04:15.929063Z","shell.execute_reply.started":"2022-09-21T21:04:14.381164Z","shell.execute_reply":"2022-09-21T21:04:15.928218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x_train1,x_train2, x_train3,x_train4","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:15.930618Z","iopub.execute_input":"2022-09-21T21:04:15.931066Z","iopub.status.idle":"2022-09-21T21:04:16.059982Z","shell.execute_reply.started":"2022-09-21T21:04:15.930878Z","shell.execute_reply":"2022-09-21T21:04:16.059327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(x_train))\nprint(len(y_train))","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:16.062932Z","iopub.execute_input":"2022-09-21T21:04:16.063194Z","iopub.status.idle":"2022-09-21T21:04:16.067791Z","shell.execute_reply.started":"2022-09-21T21:04:16.063147Z","shell.execute_reply":"2022-09-21T21:04:16.067032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)\ny_train_multi[:, 4] = y_train[:, 4]\n\nfor i in range(3, -1, -1):\n    y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])\n\nprint(\"Original y_train:\", y_train.sum(axis=0))\nprint(\"Multilabel version:\", y_train_multi.sum(axis=0))\nprint(y_train)\nprint(y_train_multi)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:16.069203Z","iopub.execute_input":"2022-09-21T21:04:16.069750Z","iopub.status.idle":"2022-09-21T21:04:16.085640Z","shell.execute_reply.started":"2022-09-21T21:04:16.069700Z","shell.execute_reply":"2022-09-21T21:04:16.084804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating multilabel**","metadata":{}},{"cell_type":"markdown","source":"Now we can split it into a training and validation set.","metadata":{}},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(\n    x_train, y_train_multi, \n    test_size=0.15, \n    random_state=2019\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:16.087670Z","iopub.execute_input":"2022-09-21T21:04:16.087968Z","iopub.status.idle":"2022-09-21T21:04:17.644746Z","shell.execute_reply.started":"2022-09-21T21:04:16.087920Z","shell.execute_reply":"2022-09-21T21:04:17.643814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Generator (Data augmentation)**\n\n","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 64\n\ndef create_datagen():\n    return ImageDataGenerator(\n        #rescale=1./255,\n        zoom_range=0.15,  # set range for random zoom\n        # set mode for filling points outside the input boundaries\n        fill_mode='constant',\n        cval=0.,  # value used for fill_mode = \"constant\"\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=True,  # randomly flip images\n    )\n\n# Using original generator\ndata_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019)\n#val_generator = create_datagen().flow(x_val,y_val, batch_size=BATCH_SIZE, seed=2019)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:17.646104Z","iopub.execute_input":"2022-09-21T21:04:17.646411Z","iopub.status.idle":"2022-09-21T21:04:22.260849Z","shell.execute_reply.started":"2022-09-21T21:04:17.646351Z","shell.execute_reply":"2022-09-21T21:04:22.259944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x_train","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:22.262201Z","iopub.execute_input":"2022-09-21T21:04:22.262494Z","iopub.status.idle":"2022-09-21T21:04:22.375945Z","shell.execute_reply.started":"2022-09-21T21:04:22.262448Z","shell.execute_reply":"2022-09-21T21:04:22.374194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import layers\nfrom tensorflow.keras.applications import DenseNet121\nfrom tensorflow.keras.callbacks import Callback, ModelCheckpoint\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:22.377706Z","iopub.execute_input":"2022-09-21T21:04:22.378289Z","iopub.status.idle":"2022-09-21T21:04:22.389052Z","shell.execute_reply.started":"2022-09-21T21:04:22.378206Z","shell.execute_reply":"2022-09-21T21:04:22.388117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_tensor = layers.Input(shape=(224,224,3))\ndensenet = DenseNet121(\n    weights=\"../input/densenet-keras/DenseNet-BC-121-32-no-top.h5\" ,\n    include_top=False,\n    input_tensor=input_tensor\n)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:22.390774Z","iopub.execute_input":"2022-09-21T21:04:22.391250Z","iopub.status.idle":"2022-09-21T21:04:39.616512Z","shell.execute_reply.started":"2022-09-21T21:04:22.391034Z","shell.execute_reply":"2022-09-21T21:04:39.615595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import Model\n\ndef build_model_functional():\n    x = layers.GlobalAveragePooling2D()(densenet.output)\n    x= layers.Dense(256, activation=\"relu\")(x)\n    x = layers.Dropout(0.5)(x)\n    final_output = layers.Dense(5, activation='sigmoid', name='final_output')(x)\n    \n    model = Model(input_tensor, final_output)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:39.618040Z","iopub.execute_input":"2022-09-21T21:04:39.618377Z","iopub.status.idle":"2022-09-21T21:04:39.626915Z","shell.execute_reply.started":"2022-09-21T21:04:39.618303Z","shell.execute_reply":"2022-09-21T21:04:39.626017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model: DenseNet-121","metadata":{}},{"cell_type":"code","source":"model = build_model_functional()\n#model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:39.628828Z","iopub.execute_input":"2022-09-21T21:04:39.629486Z","iopub.status.idle":"2022-09-21T21:04:39.748564Z","shell.execute_reply.started":"2022-09-21T21:04:39.629434Z","shell.execute_reply":"2022-09-21T21:04:39.747897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import backend as K\n\ndef recall_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef precision_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef f1_m(y_true, y_pred):\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:39.750287Z","iopub.execute_input":"2022-09-21T21:04:39.750734Z","iopub.status.idle":"2022-09-21T21:04:39.759617Z","shell.execute_reply.started":"2022-09-21T21:04:39.750562Z","shell.execute_reply":"2022-09-21T21:04:39.758818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfilepath='my_best_model.hdf5'\nrlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=3, factor=0.5, min_lr=1e-6, verbose=1)\ncheckpoint = ModelCheckpoint(filepath=filepath, \n                             monitor='val_acc',\n                             verbose=1, \n                             save_best_only=True,\n                             mode='max')\n\ncallback_list = [rlrop, checkpoint]","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:39.761414Z","iopub.execute_input":"2022-09-21T21:04:39.761852Z","iopub.status.idle":"2022-09-21T21:04:39.770306Z","shell.execute_reply.started":"2022-09-21T21:04:39.761802Z","shell.execute_reply":"2022-09-21T21:04:39.769197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001), \n        loss='binary_crossentropy', \n        metrics=['accuracy','AUC',f1_m,precision_m, recall_m]\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:39.771636Z","iopub.execute_input":"2022-09-21T21:04:39.772249Z","iopub.status.idle":"2022-09-21T21:04:40.092011Z","shell.execute_reply.started":"2022-09-21T21:04:39.771914Z","shell.execute_reply":"2022-09-21T21:04:40.091296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training & Evaluation","metadata":{}},{"cell_type":"code","source":"bucket_num = 3\ndiv = round(train_df.shape[0]/bucket_num)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:40.093409Z","iopub.execute_input":"2022-09-21T21:04:40.093688Z","iopub.status.idle":"2022-09-21T21:04:40.098354Z","shell.execute_reply.started":"2022-09-21T21:04:40.093642Z","shell.execute_reply":"2022-09-21T21:04:40.097520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_init = {\n    'val_loss': [0.0],\n    'val_acc': [0.0],\n    'loss': [0.0], \n    'acc': [0.0],\n    'bucket': [0.0]\n}\nresults = pd.DataFrame(df_init)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:40.099713Z","iopub.execute_input":"2022-09-21T21:04:40.100217Z","iopub.status.idle":"2022-09-21T21:04:40.111319Z","shell.execute_reply.started":"2022-09-21T21:04:40.100168Z","shell.execute_reply":"2022-09-21T21:04:40.110623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport gc\nepochs = [5,5,5]","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:40.114020Z","iopub.execute_input":"2022-09-21T21:04:40.114494Z","iopub.status.idle":"2022-09-21T21:04:40.119523Z","shell.execute_reply.started":"2022-09-21T21:04:40.114292Z","shell.execute_reply":"2022-09-21T21:04:40.118736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(0,bucket_num):\n    if i != (bucket_num-1):\n        print(\"Bucket Nr: {}\".format(i))\n        \n        history = model.fit_generator(\n                        data_generator,\n                        steps_per_epoch=data_generator.n//(data_generator.batch_size),\n                        epochs=epochs[i],\n                        validation_data=(x_val, y_val),\n                        callbacks=callback_list,\n                        )\n        \n        dic = history.history\n        df_model = pd.DataFrame(dic)\n        df_model['bucket'] = i\n    else:\n        print(\"Bucket Nr: {}\".format(i))\n        \n        history = model.fit_generator(\n                        data_generator,\n                        steps_per_epoch=data_generator.n//(data_generator.batch_size),\n                        epochs=epochs[i],\n                        validation_data=(x_val, y_val),\n                        callbacks=callback_list,\n                        )\n        \n        dic = history.history\n        df_model = pd.DataFrame(dic)\n        df_model['bucket'] = i\n\n    results = results.append(df_model)\n    \n    gc.collect()\n    \n    print('-'*40)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:04:40.121059Z","iopub.execute_input":"2022-09-21T21:04:40.121569Z","iopub.status.idle":"2022-09-21T21:48:22.378565Z","shell.execute_reply.started":"2022-09-21T21:04:40.121393Z","shell.execute_reply":"2022-09-21T21:48:22.377699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(results)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:22.380124Z","iopub.execute_input":"2022-09-21T21:48:22.380665Z","iopub.status.idle":"2022-09-21T21:48:22.395460Z","shell.execute_reply.started":"2022-09-21T21:48:22.380612Z","shell.execute_reply":"2022-09-21T21:48:22.394578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(y_val)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:22.400748Z","iopub.execute_input":"2022-09-21T21:48:22.401013Z","iopub.status.idle":"2022-09-21T21:48:22.405484Z","shell.execute_reply.started":"2022-09-21T21:48:22.400963Z","shell.execute_reply":"2022-09-21T21:48:22.404541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(x_val) ","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:22.407587Z","iopub.execute_input":"2022-09-21T21:48:22.408122Z","iopub.status.idle":"2022-09-21T21:48:29.824188Z","shell.execute_reply.started":"2022-09-21T21:48:22.408056Z","shell.execute_reply":"2022-09-21T21:48:29.823183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(preds)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:29.825647Z","iopub.execute_input":"2022-09-21T21:48:29.825957Z","iopub.status.idle":"2022-09-21T21:48:29.833051Z","shell.execute_reply.started":"2022-09-21T21:48:29.825905Z","shell.execute_reply":"2022-09-21T21:48:29.832129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=[]\nfor sample in  preds:\n    y_pred.append([1 if i>=0.5 else 0 for i in sample ] )\ny_pred = np.array(y_pred)\nprint(y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:29.834667Z","iopub.execute_input":"2022-09-21T21:48:29.835267Z","iopub.status.idle":"2022-09-21T21:48:29.865064Z","shell.execute_reply.started":"2022-09-21T21:48:29.835216Z","shell.execute_reply":"2022-09-21T21:48:29.863967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_val, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:29.866811Z","iopub.execute_input":"2022-09-21T21:48:29.867335Z","iopub.status.idle":"2022-09-21T21:48:29.882974Z","shell.execute_reply.started":"2022-09-21T21:48:29.867142Z","shell.execute_reply":"2022-09-21T21:48:29.881955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import multilabel_confusion_matrix\nmultilabel_confusion_matrix(y_val, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:29.884389Z","iopub.execute_input":"2022-09-21T21:48:29.884918Z","iopub.status.idle":"2022-09-21T21:48:29.896361Z","shell.execute_reply.started":"2022-09-21T21:48:29.884625Z","shell.execute_reply":"2022-09-21T21:48:29.895402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\nprint(classification_report(y_val, y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:29.897676Z","iopub.execute_input":"2022-09-21T21:48:29.899141Z","iopub.status.idle":"2022-09-21T21:48:29.921164Z","shell.execute_reply.started":"2022-09-21T21:48:29.897920Z","shell.execute_reply":"2022-09-21T21:48:29.920224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:29.922497Z","iopub.execute_input":"2022-09-21T21:48:29.922943Z","iopub.status.idle":"2022-09-21T21:48:29.927206Z","shell.execute_reply.started":"2022-09-21T21:48:29.922738Z","shell.execute_reply":"2022-09-21T21:48:29.926256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"micro: {:.2f}\".format(metrics.average_precision_score(y_val, preds, average='micro')))\nprint(\"macro: {:.2f} \".format( metrics.average_precision_score(y_val, preds, average='macro')))\nprint(\"weighted: {:.2f} \".format( metrics.average_precision_score(y_val, preds, average='weighted')))\nprint(\"samples: {:.2f} \".format( metrics.average_precision_score(y_val, preds, average='samples')))  ","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:29.928461Z","iopub.execute_input":"2022-09-21T21:48:29.928885Z","iopub.status.idle":"2022-09-21T21:48:30.480784Z","shell.execute_reply.started":"2022-09-21T21:48:29.928706Z","shell.execute_reply":"2022-09-21T21:48:30.479862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"F1 of each label: {}\".format(metrics.f1_score(y_val, y_pred, average=None)))","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:30.482150Z","iopub.execute_input":"2022-09-21T21:48:30.482444Z","iopub.status.idle":"2022-09-21T21:48:30.493512Z","shell.execute_reply.started":"2022-09-21T21:48:30.482394Z","shell.execute_reply":"2022-09-21T21:48:30.492606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"micro: {:.2f}\".format(metrics.f1_score(y_val, y_pred, average='micro')))\nprint(\"macro: {:.2f} \".format( metrics.f1_score(y_val, y_pred, average='macro')))\nprint(\"weighted: {:.2f} \".format( metrics.f1_score(y_val, y_pred, average='weighted')))\nprint(\"samples: {:.2f} \".format( metrics.f1_score(y_val, y_pred, average='samples')))  ","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:30.494867Z","iopub.execute_input":"2022-09-21T21:48:30.495330Z","iopub.status.idle":"2022-09-21T21:48:30.518499Z","shell.execute_reply.started":"2022-09-21T21:48:30.495278Z","shell.execute_reply":"2022-09-21T21:48:30.517622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"micro: {:.2f}\".format(metrics.recall_score(y_val, y_pred, average='micro')))\nprint(\"macro: {:.2f} \".format( metrics.recall_score(y_val, y_pred, average='macro')))\nprint(\"weighted: {:.2f} \".format( metrics.recall_score(y_val, y_pred, average='weighted')))\nprint(\"samples: {:.2f} \".format( metrics.recall_score(y_val, y_pred, average='samples'))) ","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:30.519916Z","iopub.execute_input":"2022-09-21T21:48:30.520399Z","iopub.status.idle":"2022-09-21T21:48:30.543042Z","shell.execute_reply.started":"2022-09-21T21:48:30.520345Z","shell.execute_reply":"2022-09-21T21:48:30.542332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selectedLabel1=0\nselectedLabel2=1\nselectedLabel3=2\nselectedLabel4=3\nselectedLabel5=4\nprint(\"y_true\\n\", y_val)\ny_binary1 = y_val[:,selectedLabel1]\ny_binary2 = y_val[:,selectedLabel2]\ny_binary3 = y_val[:,selectedLabel3]\ny_binary4 = y_val[:,selectedLabel4]\ny_binary5 = y_val[:,selectedLabel5]","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:30.544230Z","iopub.execute_input":"2022-09-21T21:48:30.544675Z","iopub.status.idle":"2022-09-21T21:48:30.552528Z","shell.execute_reply.started":"2022-09-21T21:48:30.544626Z","shell.execute_reply":"2022-09-21T21:48:30.551628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_binary_scores1 = preds[:,selectedLabel1]\ny_binary_scores2 = preds[:,selectedLabel2]\ny_binary_scores3 = preds[:,selectedLabel3]\ny_binary_scores4 = preds[:,selectedLabel4]\ny_binary_scores5 = preds[:,selectedLabel5]","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:30.554009Z","iopub.execute_input":"2022-09-21T21:48:30.554380Z","iopub.status.idle":"2022-09-21T21:48:30.560237Z","shell.execute_reply.started":"2022-09-21T21:48:30.554314Z","shell.execute_reply":"2022-09-21T21:48:30.559220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve\n\nfpr1, tpr1, thresholds1 = roc_curve(y_binary1, y_binary_scores1)\nfpr2, tpr2, thresholds2 = roc_curve(y_binary2, y_binary_scores2)\nfpr3, tpr3, thresholds3 = roc_curve(y_binary3, y_binary_scores3)\nfpr4, tpr4, thresholds4 = roc_curve(y_binary4, y_binary_scores4)\nfpr5, tpr5, thresholds5 = roc_curve(y_binary5, y_binary_scores5)","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:30.561776Z","iopub.execute_input":"2022-09-21T21:48:30.562222Z","iopub.status.idle":"2022-09-21T21:48:30.576466Z","shell.execute_reply.started":"2022-09-21T21:48:30.562141Z","shell.execute_reply":"2022-09-21T21:48:30.575602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure()\nlw = 1\nplt.plot(fpr1, tpr1,\n         lw=lw, label='ROC curve 1' )\nplt.plot(fpr2, tpr2,\n         lw=lw, label='ROC curve 2' )\nplt.plot(fpr3, tpr3,\n         lw=lw, label='ROC curve 3' )\nplt.plot(fpr4, tpr4,\n         lw=lw, label='ROC curve 4' )\nplt.plot(fpr5, tpr5,\n         lw=lw, label='ROC curve 5' )\nplt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\n#plt.title('Receiver operating characteristic for {}'.format(selectedLabel))\nplt.legend(loc=\"lower right\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:30.577755Z","iopub.execute_input":"2022-09-21T21:48:30.578259Z","iopub.status.idle":"2022-09-21T21:48:30.748632Z","shell.execute_reply.started":"2022-09-21T21:48:30.578209Z","shell.execute_reply":"2022-09-21T21:48:30.747829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_names = ['label 0', 'label 1', 'label 3', 'label 4', 'label 5']","metadata":{"execution":{"iopub.status.busy":"2022-09-21T21:48:30.749826Z","iopub.execute_input":"2022-09-21T21:48:30.750292Z","iopub.status.idle":"2022-09-21T21:48:30.755162Z","shell.execute_reply.started":"2022-09-21T21:48:30.750236Z","shell.execute_reply":"2022-09-21T21:48:30.754155Z"},"trusted":true},"execution_count":null,"outputs":[]}]}