{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"","metadata":{}},{"cell_type":"code","source":"import json\nimport math\nimport os\n\nimport cv2\nfrom PIL import Image\nimport numpy as np\nfrom keras import layers\nfrom keras.applications import DenseNet121\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\nimport scipy\nimport tensorflow as tf\nfrom tqdm import tqdm\nfrom scipy import ndimage, misc\n\n%matplotlib inline","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-09-26T10:08:22.402877Z","iopub.execute_input":"2022-09-26T10:08:22.403190Z","iopub.status.idle":"2022-09-26T10:08:25.718113Z","shell.execute_reply.started":"2022-09-26T10:08:22.403141Z","shell.execute_reply":"2022-09-26T10:08:25.717041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Set random seed for reproducibility.","metadata":{}},{"cell_type":"code","source":"np.random.seed(2019)\ntf.set_random_seed(2019)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:25.720583Z","iopub.execute_input":"2022-09-26T10:08:25.720950Z","iopub.status.idle":"2022-09-26T10:08:25.757137Z","shell.execute_reply.started":"2022-09-26T10:08:25.720894Z","shell.execute_reply":"2022-09-26T10:08:25.756200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\ntest_df = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\nprint(train_df.shape)\nprint(test_df.shape)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:25.759012Z","iopub.execute_input":"2022-09-26T10:08:25.759472Z","iopub.status.idle":"2022-09-26T10:08:25.806512Z","shell.execute_reply.started":"2022-09-26T10:08:25.759407Z","shell.execute_reply":"2022-09-26T10:08:25.805730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['diagnosis'].hist()\ntrain_df['diagnosis'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:25.807960Z","iopub.execute_input":"2022-09-26T10:08:25.808282Z","iopub.status.idle":"2022-09-26T10:08:26.081715Z","shell.execute_reply.started":"2022-09-26T10:08:25.808229Z","shell.execute_reply":"2022-09-26T10:08:26.080836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Displaying some Sample Images","metadata":{}},{"cell_type":"markdown","source":"We will resize the images to 224x224","metadata":{}},{"cell_type":"code","source":"def unsharp_mask(image, kernel_size=(5, 5), sigma=1.0, amount=1.5, threshold=0):\n    \"\"\"Return a sharpened version of the image, using an unsharp mask.\"\"\"\n    blurred = cv2.GaussianBlur(image, kernel_size, sigma)\n    sharpened = float(amount + 1) * image - float(amount) * blurred\n    sharpened = np.maximum(sharpened, np.zeros(sharpened.shape))\n    sharpened = np.minimum(sharpened, 255 * np.ones(sharpened.shape))\n    sharpened = sharpened.round().astype(np.uint8)\n    if threshold > 0:\n        low_contrast_mask = np.absolute(image - blurred) < threshold\n        np.copyto(sharpened, image, where=low_contrast_mask)\n    return sharpened\n\ndef CLAHEgreen(image):\n    green=image[:, :, 1]\n    clipLimit = 2.0\n    tileGridSize = (8,8)\n    clahe=cv2.createCLAHE(clipLimit = clipLimit, tileGridSize = tileGridSize)\n    cla=clahe.apply(green)\n    img=cv2.merge((cla,cla,cla))\n    \n    return img\n\ndef preprocess_image(path):\n    image = cv2.imread(path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    #image = cv2.resize(image, (224, 224))\n    #image=CLAHEgreen(image)\n    image_sharp=unsharp_mask(image)\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:26.085254Z","iopub.execute_input":"2022-09-26T10:08:26.085760Z","iopub.status.idle":"2022-09-26T10:08:26.097141Z","shell.execute_reply.started":"2022-09-26T10:08:26.085700Z","shell.execute_reply":"2022-09-26T10:08:26.096107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_image1(path):\n    img = cv2.imread(path)\n    \n    #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    #img = cv2.resize(img, (224,224))\n    #image=CLAHEgreen(img)\n    #image_sharp=unsharp_mask(image)\n    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    result1 = ndimage.gaussian_laplace(img, sigma=3)\n    result2=zero_crossing(result1)\n    result3 = cv2.merge((result2,result2,result2))\n    return result3\ndef any_neighbor_zero(img, i, j):\n    for k in range(-1,2):\n        for l in range(-1,2):\n            if img[i+k, j+k] == 0:\n                return True\n    return False\n\ndef zero_crossing(img):\n    img[img > 0] = 1\n    img[img < 0] = 0\n    out_img = np.zeros(img.shape)\n    for i in range(1,img.shape[0]-1):\n        for j in range(1,img.shape[1]-1):\n            if img[i,j] > 0 and any_neighbor_zero(img, i, j):\n                out_img[i,j] = 255\n    return out_img","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:26.100816Z","iopub.execute_input":"2022-09-26T10:08:26.101379Z","iopub.status.idle":"2022-09-26T10:08:26.117178Z","shell.execute_reply.started":"2022-09-26T10:08:26.101155Z","shell.execute_reply":"2022-09-26T10:08:26.115881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = train_df.shape[0]\nx_train = np.empty((N, 224, 224, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate(tqdm(train_df['id_code'])):\n    x_train[i, :, :, :] = preprocess_image(\n        f'../input/cropped-degrees-dataset/cropped 0.8/{image_id}.jpg'\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:26.119634Z","iopub.execute_input":"2022-09-26T10:08:26.120324Z","iopub.status.idle":"2022-09-26T10:08:52.466252Z","shell.execute_reply.started":"2022-09-26T10:08:26.120159Z","shell.execute_reply":"2022-09-26T10:08:52.465336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = pd.get_dummies(train_df['diagnosis']).values\n","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:52.467720Z","iopub.execute_input":"2022-09-26T10:08:52.468259Z","iopub.status.idle":"2022-09-26T10:08:52.479323Z","shell.execute_reply.started":"2022-09-26T10:08:52.468201Z","shell.execute_reply":"2022-09-26T10:08:52.478234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)\ny_train_multi[:, 4] = y_train[:, 4]\n\nfor i in range(3, -1, -1):\n    y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])\n\nprint(\"Original y_train:\", y_train.sum(axis=0))\nprint(\"Multilabel version:\", y_train_multi.sum(axis=0))\nprint(y_train)\nprint(y_train_multi)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:52.481563Z","iopub.execute_input":"2022-09-26T10:08:52.482143Z","iopub.status.idle":"2022-09-26T10:08:52.496397Z","shell.execute_reply.started":"2022-09-26T10:08:52.482000Z","shell.execute_reply":"2022-09-26T10:08:52.495552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating multilabel**","metadata":{}},{"cell_type":"markdown","source":"Now we can split it into a training and validation set.","metadata":{}},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(\n    x_train, y_train_multi, \n    test_size=0.15, \n    random_state=2019\n)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:52.497975Z","iopub.execute_input":"2022-09-26T10:08:52.498473Z","iopub.status.idle":"2022-09-26T10:08:52.953407Z","shell.execute_reply.started":"2022-09-26T10:08:52.498260Z","shell.execute_reply":"2022-09-26T10:08:52.952471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for i in tqdm(range(3112)):\n#    x_train[i,:,:,:] =preprocess_image1(x_train[i,:,:,:])","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:52.955806Z","iopub.execute_input":"2022-09-26T10:08:52.956661Z","iopub.status.idle":"2022-09-26T10:08:52.962575Z","shell.execute_reply.started":"2022-09-26T10:08:52.956349Z","shell.execute_reply":"2022-09-26T10:08:52.960747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Generator (Data augmentation)**\n\n","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 64\n\ndef create_datagen():\n    return ImageDataGenerator(\n        #rescale=1./255,\n        zoom_range=0.15,  # set range for random zoom\n        # set mode for filling points outside the input boundaries\n        fill_mode='constant',\n        cval=0.,  # value used for fill_mode = \"constant\"\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=True,  # randomly flip images\n    )\n\n# Using original generator\ndata_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019)\nval_generator = create_datagen().flow(x_val,y_val, batch_size=BATCH_SIZE, seed=2019)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:52.964891Z","iopub.execute_input":"2022-09-26T10:08:52.965500Z","iopub.status.idle":"2022-09-26T10:08:54.693437Z","shell.execute_reply.started":"2022-09-26T10:08:52.965232Z","shell.execute_reply":"2022-09-26T10:08:54.692426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import layers\nfrom tensorflow.keras.applications import DenseNet121\nfrom tensorflow.keras.callbacks import Callback, ModelCheckpoint\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:54.700430Z","iopub.execute_input":"2022-09-26T10:08:54.703910Z","iopub.status.idle":"2022-09-26T10:08:54.717990Z","shell.execute_reply.started":"2022-09-26T10:08:54.703748Z","shell.execute_reply":"2022-09-26T10:08:54.716748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_tensor = layers.Input(shape=(224,224,3))\ndensenet = DenseNet121(\n    weights=\"../input/densenet-keras/DenseNet-BC-121-32-no-top.h5\" ,\n    include_top=False,\n    input_tensor=input_tensor\n)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:08:54.723094Z","iopub.execute_input":"2022-09-26T10:08:54.725497Z","iopub.status.idle":"2022-09-26T10:09:16.042860Z","shell.execute_reply.started":"2022-09-26T10:08:54.725435Z","shell.execute_reply":"2022-09-26T10:09:16.041913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import Model\n\ndef build_model_functional():\n    x = layers.GlobalAveragePooling2D()(densenet.output)\n    x= layers.Dense(256, activation=\"relu\")(x)\n    x = layers.Dropout(0.5)(x)\n    final_output = layers.Dense(5, activation='sigmoid', name='final_output')(x)\n    \n    model = Model(input_tensor, final_output)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:09:16.044849Z","iopub.execute_input":"2022-09-26T10:09:16.045468Z","iopub.status.idle":"2022-09-26T10:09:16.052824Z","shell.execute_reply.started":"2022-09-26T10:09:16.045410Z","shell.execute_reply":"2022-09-26T10:09:16.052224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model: DenseNet-121","metadata":{}},{"cell_type":"code","source":"model = build_model_functional()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:09:16.054515Z","iopub.execute_input":"2022-09-26T10:09:16.055230Z","iopub.status.idle":"2022-09-26T10:09:16.485714Z","shell.execute_reply.started":"2022-09-26T10:09:16.055170Z","shell.execute_reply":"2022-09-26T10:09:16.482173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import backend as K\n\ndef recall_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef precision_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef f1_m(y_true, y_pred):\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:09:16.490828Z","iopub.execute_input":"2022-09-26T10:09:16.491186Z","iopub.status.idle":"2022-09-26T10:09:16.515614Z","shell.execute_reply.started":"2022-09-26T10:09:16.491123Z","shell.execute_reply":"2022-09-26T10:09:16.514626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfilepath='my_best_model.h5'\nrlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=3, factor=0.5, min_lr=1e-6, verbose=1)\ncheckpoint = ModelCheckpoint(filepath=filepath, \n                             monitor='val_acc',\n                             verbose=1, \n                             save_best_only=True,\n                             mode='max')\n\ncallback_list = [rlrop, checkpoint]","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:09:16.517213Z","iopub.execute_input":"2022-09-26T10:09:16.517615Z","iopub.status.idle":"2022-09-26T10:09:16.538984Z","shell.execute_reply.started":"2022-09-26T10:09:16.517554Z","shell.execute_reply":"2022-09-26T10:09:16.533400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n        optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001), \n        loss='binary_crossentropy', \n        metrics=['accuracy','AUC',f1_m,precision_m, recall_m]\n    )","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:09:16.541517Z","iopub.execute_input":"2022-09-26T10:09:16.541847Z","iopub.status.idle":"2022-09-26T10:09:17.076857Z","shell.execute_reply.started":"2022-09-26T10:09:16.541797Z","shell.execute_reply":"2022-09-26T10:09:17.076019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training & Evaluation","metadata":{}},{"cell_type":"code","source":"bucket_num = 3\ndiv = round(train_df.shape[0]/bucket_num)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:09:17.078439Z","iopub.execute_input":"2022-09-26T10:09:17.078782Z","iopub.status.idle":"2022-09-26T10:09:17.084179Z","shell.execute_reply.started":"2022-09-26T10:09:17.078709Z","shell.execute_reply":"2022-09-26T10:09:17.083049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_init = {\n    'val_loss': [0.0],\n    'val_acc': [0.0],\n    'loss': [0.0], \n    'acc': [0.0],\n    'bucket': [0.0]\n}\nresults = pd.DataFrame(df_init)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:09:17.085638Z","iopub.execute_input":"2022-09-26T10:09:17.086264Z","iopub.status.idle":"2022-09-26T10:09:17.095730Z","shell.execute_reply.started":"2022-09-26T10:09:17.086208Z","shell.execute_reply":"2022-09-26T10:09:17.094836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport gc\nepochs = [5,5,5]","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:09:17.098979Z","iopub.execute_input":"2022-09-26T10:09:17.099407Z","iopub.status.idle":"2022-09-26T10:09:17.109279Z","shell.execute_reply.started":"2022-09-26T10:09:17.099328Z","shell.execute_reply":"2022-09-26T10:09:17.107719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(0,bucket_num):\n    if i != (bucket_num-1):\n        print(\"Bucket Nr: {}\".format(i))\n        \n        history = model.fit_generator(\n                        data_generator,\n                        steps_per_epoch=data_generator.n//data_generator.batch_size,\n                        epochs=epochs[i],\n                        validation_data=(x_val, y_val),\n                        callbacks=callback_list,\n                        )\n        \n        dic = history.history\n        df_model = pd.DataFrame(dic)\n        df_model['bucket'] = i\n    else:\n        print(\"Bucket Nr: {}\".format(i))\n        \n        history = model.fit_generator(\n                        data_generator,\n                        steps_per_epoch=data_generator.n//data_generator.batch_size,\n                        epochs=epochs[i],\n                        validation_data=(x_val, y_val),\n                        callbacks=callback_list,\n                        )\n        \n        dic = history.history\n        df_model = pd.DataFrame(dic)\n        df_model['bucket'] = i\n\n    results = results.append(df_model)\n    \n    gc.collect()\n    \n    print('-'*40)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:09:17.111371Z","iopub.execute_input":"2022-09-26T10:09:17.112026Z","iopub.status.idle":"2022-09-26T10:22:50.448138Z","shell.execute_reply.started":"2022-09-26T10:09:17.111703Z","shell.execute_reply":"2022-09-26T10:22:50.446677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(results)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:22:50.450383Z","iopub.execute_input":"2022-09-26T10:22:50.450814Z","iopub.status.idle":"2022-09-26T10:22:50.478072Z","shell.execute_reply.started":"2022-09-26T10:22:50.450729Z","shell.execute_reply":"2022-09-26T10:22:50.477038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import load_model\ndependencies = {\n    'f1_m': f1_m,\n    'precision_m' : precision_m,\n    'recall_m' : recall_m\n}\nmodel=load_model('/kaggle/working/my_best_model.h5', custom_objects=dependencies)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:22:50.480892Z","iopub.execute_input":"2022-09-26T10:22:50.482468Z","iopub.status.idle":"2022-09-26T10:23:45.158720Z","shell.execute_reply.started":"2022-09-26T10:22:50.482394Z","shell.execute_reply":"2022-09-26T10:23:45.157793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x_val)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:23:45.164511Z","iopub.execute_input":"2022-09-26T10:23:45.164792Z","iopub.status.idle":"2022-09-26T10:23:45.172575Z","shell.execute_reply.started":"2022-09-26T10:23:45.164729Z","shell.execute_reply":"2022-09-26T10:23:45.171802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(x_val) \nprint(preds)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:23:45.174043Z","iopub.execute_input":"2022-09-26T10:23:45.174548Z","iopub.status.idle":"2022-09-26T10:24:07.176423Z","shell.execute_reply.started":"2022-09-26T10:23:45.174496Z","shell.execute_reply":"2022-09-26T10:24:07.175328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=[]\nfor sample in  preds:\n    y_pred.append([1 if i>=0.5 else 0 for i in sample ] )\ny_pred = np.array(y_pred)\nprint(y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:24:07.177874Z","iopub.execute_input":"2022-09-26T10:24:07.178205Z","iopub.status.idle":"2022-09-26T10:24:07.196131Z","shell.execute_reply.started":"2022-09-26T10:24:07.178153Z","shell.execute_reply":"2022-09-26T10:24:07.195150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_val, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:24:07.197700Z","iopub.execute_input":"2022-09-26T10:24:07.198334Z","iopub.status.idle":"2022-09-26T10:24:07.210839Z","shell.execute_reply.started":"2022-09-26T10:24:07.198278Z","shell.execute_reply":"2022-09-26T10:24:07.209542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import multilabel_confusion_matrix\nmultilabel_confusion_matrix(y_val, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:24:07.212413Z","iopub.execute_input":"2022-09-26T10:24:07.212957Z","iopub.status.idle":"2022-09-26T10:24:07.229178Z","shell.execute_reply.started":"2022-09-26T10:24:07.212713Z","shell.execute_reply":"2022-09-26T10:24:07.227744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\nprint(classification_report(y_val, y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:24:07.231013Z","iopub.execute_input":"2022-09-26T10:24:07.231643Z","iopub.status.idle":"2022-09-26T10:24:07.279807Z","shell.execute_reply.started":"2022-09-26T10:24:07.231582Z","shell.execute_reply":"2022-09-26T10:24:07.276316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selectedLabel1=0\nselectedLabel2=1\nselectedLabel3=2\nselectedLabel4=3\nselectedLabel5=4\nprint(\"y_true\\n\", y_val)\ny_binary1 = y_val[:,selectedLabel1]\ny_binary2 = y_val[:,selectedLabel2]\ny_binary3 = y_val[:,selectedLabel3]\ny_binary4 = y_val[:,selectedLabel4]\ny_binary5 = y_val[:,selectedLabel5]","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:24:07.295127Z","iopub.execute_input":"2022-09-26T10:24:07.295626Z","iopub.status.idle":"2022-09-26T10:24:07.318496Z","shell.execute_reply.started":"2022-09-26T10:24:07.295557Z","shell.execute_reply":"2022-09-26T10:24:07.316948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_binary_scores1 = preds[:,selectedLabel1]\ny_binary_scores2 = preds[:,selectedLabel2]\ny_binary_scores3 = preds[:,selectedLabel3]\ny_binary_scores4 = preds[:,selectedLabel4]\ny_binary_scores5 = preds[:,selectedLabel5]\n","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:24:07.323530Z","iopub.execute_input":"2022-09-26T10:24:07.323932Z","iopub.status.idle":"2022-09-26T10:24:07.339130Z","shell.execute_reply.started":"2022-09-26T10:24:07.323868Z","shell.execute_reply":"2022-09-26T10:24:07.337025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve\nfpr1, tpr1, thresholds1 = roc_curve(y_binary1, y_binary_scores1)\nfpr2, tpr2, thresholds2 = roc_curve(y_binary2, y_binary_scores2)\nfpr3, tpr3, thresholds3 = roc_curve(y_binary3, y_binary_scores3)\nfpr4, tpr4, thresholds4 = roc_curve(y_binary4, y_binary_scores4)\nfpr5, tpr5, thresholds5 = roc_curve(y_binary5, y_binary_scores5)","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:24:07.344991Z","iopub.execute_input":"2022-09-26T10:24:07.346075Z","iopub.status.idle":"2022-09-26T10:24:07.365466Z","shell.execute_reply.started":"2022-09-26T10:24:07.345970Z","shell.execute_reply":"2022-09-26T10:24:07.364524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure()\nlw = 1\nplt.plot(fpr1, tpr1,\n         lw=lw, label='ROC curve 1' )\nplt.plot(fpr2, tpr2,\n         lw=lw, label='ROC curve 2' )\nplt.plot(fpr3, tpr3,\n         lw=lw, label='ROC curve 3' )\nplt.plot(fpr4, tpr4,\n         lw=lw, label='ROC curve 4' )\nplt.plot(fpr5, tpr5,\n         lw=lw, label='ROC curve 5' )\nplt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver operating characteristic for {}'.format(selectedLabel1))\nplt.legend(loc=\"lower right\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-26T10:24:07.366983Z","iopub.execute_input":"2022-09-26T10:24:07.367509Z","iopub.status.idle":"2022-09-26T10:24:07.708398Z","shell.execute_reply.started":"2022-09-26T10:24:07.367451Z","shell.execute_reply":"2022-09-26T10:24:07.707082Z"},"trusted":true},"execution_count":null,"outputs":[]}]}