{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, sys, math, json\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport skimage.io\n\nfrom skimage.transform import resize\nfrom imgaug import augmenters as iaa\nfrom tqdm import tqdm\nimport PIL\nfrom PIL import Image, ImageOps\nimport cv2\nfrom sklearn.utils import class_weight, shuffle\n\n%matplotlib inline\n\nimport scipy\nimport tensorflow as tf\n\nfrom tensorflow.keras.losses import binary_crossentropy\nfrom tensorflow.keras.applications.resnet50 import preprocess_input\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.utils import to_categorical\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.applications import DenseNet121\nfrom tensorflow.keras.callbacks import Callback, ModelCheckpoint\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\n\nfrom sklearn.metrics import f1_score, fbeta_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score, accuracy_score\n\n\nWORKERS = 2\nCHANNEL = 3\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nnp.random.seed(42)\ntf.random.set_seed(42)\n\nIMG_SIZE = 512\nNUM_CLASSES = 5\nSEED = 42\nTRAIN_NUM = 1000 # use 1000 when you just want to explore new idea, use -1 for full train\n\n#### Starting with just 2019 data","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:44:57.327767Z","iopub.execute_input":"2021-12-10T05:44:57.328614Z","iopub.status.idle":"2021-12-10T05:45:04.882779Z","shell.execute_reply.started":"2021-12-10T05:44:57.328573Z","shell.execute_reply":"2021-12-10T05:45:04.881485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train = pd.read_csv('../input/aptos2019-blindness-detection/train.csv')\n# df_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\n\ndf_train = pd.read_csv('../input/diabetic-retinopathy-resized/trainLabels.csv')\ndf_test = pd.read_csv('../input/aptos2019-blindness-detection/test.csv')\n\nprint(df_train.shape)\nprint(df_test.shape)\nprint(df_train.head())","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:45:04.88477Z","iopub.execute_input":"2021-12-10T05:45:04.885183Z","iopub.status.idle":"2021-12-10T05:45:04.964164Z","shell.execute_reply.started":"2021-12-10T05:45:04.885138Z","shell.execute_reply":"2021-12-10T05:45:04.963072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_weights = df_train['level'].value_counts()\ndfs = [df_train[df_train['level'] == i].sample(1200*(2 if i < 3 else 1),replace=True) for i in range(5)]\nresampled = pd.concat(dfs, axis = 0).reset_index(drop=True)\nresampled","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:46:53.747647Z","iopub.execute_input":"2021-12-10T05:46:53.747951Z","iopub.status.idle":"2021-12-10T05:46:53.778065Z","shell.execute_reply.started":"2021-12-10T05:46:53.747919Z","shell.execute_reply":"2021-12-10T05:46:53.777061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resampled.level.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:46:55.066346Z","iopub.execute_input":"2021-12-10T05:46:55.066649Z","iopub.status.idle":"2021-12-10T05:46:55.075891Z","shell.execute_reply.started":"2021-12-10T05:46:55.066617Z","shell.execute_reply":"2021-12-10T05:46:55.074846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=resampled","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:44:55.131308Z","iopub.status.idle":"2021-12-10T05:44:55.132144Z","shell.execute_reply.started":"2021-12-10T05:44:55.131819Z","shell.execute_reply":"2021-12-10T05:44:55.131851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### There are a number of ways to preprocess the images. Lets start by looking at them as is.","metadata":{}},{"cell_type":"code","source":"def display_samples(df, columns=4, rows=3, gauss=False):\n    fig=plt.figure(figsize=(5*columns, 4*rows))\n\n    for i in range(columns*rows):\n        image_path = df.loc[i,'image']\n        image_id = df.loc[i,'level']\n#         img = cv2.imread(f'../input/aptos2019-blindness-detection/train_images/{image_path}.png')\n        img = cv2.imread(f'../input/diabetic-retinopathy-resized/resized_train/resized_train/{image_path}.jpeg')\n\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        if gauss:\n            img = cv2.addWeighted (img,4, cv2.GaussianBlur( img , (0,0) , IMG_SIZE/10) ,-4 ,128) \n\n        fig.add_subplot(rows, columns, i+1)\n        plt.title(image_id)\n        plt.imshow(img)\n    \n    plt.tight_layout()\n    \ndisplay_samples(df_train)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:06:39.583172Z","iopub.execute_input":"2021-12-10T04:06:39.583809Z","iopub.status.idle":"2021-12-10T04:06:43.098161Z","shell.execute_reply.started":"2021-12-10T04:06:39.583773Z","shell.execute_reply":"2021-12-10T04:06:43.097342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### A trick that worked in the previous competition was to use GuassianBlur","metadata":{}},{"cell_type":"code","source":"display_samples(df_train, gauss=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:06:51.252572Z","iopub.execute_input":"2021-12-10T04:06:51.252835Z","iopub.status.idle":"2021-12-10T04:07:03.464255Z","shell.execute_reply.started":"2021-12-10T04:06:51.252805Z","shell.execute_reply":"2021-12-10T04:07:03.463495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will resize the images to 224x224, then create a single numpy array to hold the data.","metadata":{}},{"cell_type":"code","source":"def preprocess_image(image_path, desired_size=224, gauss=False):\n    im = cv2.imread(image_path)\n    im = cv2.resize(im, (desired_size, desired_size), interpolation = cv2.INTER_AREA)\n    if gauss:\n        im = cv2.addWeighted(im,4, cv2.GaussianBlur( im , (0,0) , desired_size/10) ,-4 ,128)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:07:19.925238Z","iopub.execute_input":"2021-12-10T04:07:19.925677Z","iopub.status.idle":"2021-12-10T04:07:19.933041Z","shell.execute_reply.started":"2021-12-10T04:07:19.925641Z","shell.execute_reply":"2021-12-10T04:07:19.93198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Resizing and applying blurr","metadata":{}},{"cell_type":"code","source":"N = df_train.shape[0]\nx_train_array = np.empty((N, 224, 224, 3), dtype=np.uint8)\n\nfor i, image_id in enumerate(tqdm(df_train['image'])):\n    x_train_array[i, :, :, :] = preprocess_image(\n        f'../input/diabetic-retinopathy-resized/resized_train/resized_train/{image_id}.jpeg',\n#         f'../input/diabetic-retinopathy-resized/resized_train{image_id}.png',\n        gauss=True\n    )\n#     Image.fromarray(x_train_array[i, :, :, :]).save(f'/kaggle/working/2019_244_resized_gauss/test_images/{image_id}.jpeg')","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:07:21.538171Z","iopub.execute_input":"2021-12-10T04:07:21.538853Z","iopub.status.idle":"2021-12-10T04:13:54.243523Z","shell.execute_reply.started":"2021-12-10T04:07:21.538817Z","shell.execute_reply":"2021-12-10T04:13:54.242808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets look at our new images!","metadata":{}},{"cell_type":"code","source":"fig=plt.figure(figsize=(4*4, 3*3))\nfor i in range(12):\n    fig.add_subplot(4, 3, i+1)\n    plt.imshow(x_train_array[i])","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:14:52.69618Z","iopub.execute_input":"2021-12-10T04:14:52.696754Z","iopub.status.idle":"2021-12-10T04:14:54.013584Z","shell.execute_reply.started":"2021-12-10T04:14:52.696717Z","shell.execute_reply":"2021-12-10T04:14:54.011203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = df_test.shape[0]\nx_test = np.empty((N, 224, 224, 3), dtype=np.uint8)\nfor i, image_id in enumerate(tqdm(df_test['id_code'])):\n    x_test[i, :, :, :] = preprocess_image(\n        f'../input/aptos2019-blindness-detection/test_images/{image_id}.png',\n        gauss = True\n    )\n#     Image.fromarray(x_test_array[i, :, :, :]).save(f'/kaggle/working/2019_244_resized_gauss/test_images/{image_id}.jpeg')","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:14:58.301377Z","iopub.execute_input":"2021-12-10T04:14:58.302176Z","iopub.status.idle":"2021-12-10T04:17:50.182416Z","shell.execute_reply.started":"2021-12-10T04:14:58.302109Z","shell.execute_reply":"2021-12-10T04:17:50.181709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = pd.get_dummies(df_train['level']).values\n\nprint(x_train_array.shape)\nprint(y_train.shape)\nprint(x_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:22:29.080649Z","iopub.execute_input":"2021-12-10T04:22:29.080901Z","iopub.status.idle":"2021-12-10T04:22:29.089855Z","shell.execute_reply.started":"2021-12-10T04:22:29.080871Z","shell.execute_reply":"2021-12-10T04:22:29.088923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Instead of predicting a single label, we will change our target to be a multilabel problem; i.e., if the target is a certain class, then it encompasses all the classes before it. E.g. encoding a class 4 retinopathy would usually be [0, 0, 0, 1], but in our case we will predict [1, 1, 1, 1]. For more details, please check out Lex's kernel.","metadata":{}},{"cell_type":"code","source":"y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)\ny_train_multi[:, 4] = y_train[:, 4]\n\nfor i in range(3, -1, -1):\n    y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])\n\nprint(\"Original y_train:\", y_train.sum(axis=0))\nprint(\"Multilabel version:\", y_train_multi.sum(axis=0))","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:22:40.218558Z","iopub.execute_input":"2021-12-10T04:22:40.218815Z","iopub.status.idle":"2021-12-10T04:22:40.226782Z","shell.execute_reply.started":"2021-12-10T04:22:40.218787Z","shell.execute_reply":"2021-12-10T04:22:40.225904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(\n    x_train_array, y_train_multi, \n    test_size=0.05, \n    random_state=2019\n)\n\nprint(x_train.shape, y_train.shape, x_val.shape, y_val.shape)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:22:57.306462Z","iopub.execute_input":"2021-12-10T04:22:57.306725Z","iopub.status.idle":"2021-12-10T04:22:57.596025Z","shell.execute_reply.started":"2021-12-10T04:22:57.306697Z","shell.execute_reply":"2021-12-10T04:22:57.595266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Building the data generator. It lets us do some fancy things like flip and randomly zoom the images in the training. ","metadata":{}},{"cell_type":"code","source":"BATCH_SIZE = 32\n\ndef create_datagen():\n    return ImageDataGenerator(\n        zoom_range=0.15,  # set range for random zoom\n        # set mode for filling points outside the input boundaries\n        fill_mode='constant',\n        cval=0.,  # value used for fill_mode = \"constant\"\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=True,  # randomly flip images\n    )\n\n# Using original generator\ndata_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019)\n# Using Mixup\n# mixup_generator = MixupGenerator(x_train, y_train, batch_size=BATCH_SIZE, alpha=0.2, datagen=create_datagen())()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:22:59.743349Z","iopub.execute_input":"2021-12-10T04:22:59.743853Z","iopub.status.idle":"2021-12-10T04:23:00.624723Z","shell.execute_reply.started":"2021-12-10T04:22:59.743817Z","shell.execute_reply":"2021-12-10T04:23:00.623818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"densenet = DenseNet121(\n    weights='../input/densenet-keras/DenseNet-BC-121-32-no-top.h5',\n    include_top=False,\n    input_shape=(224,224,3)\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-10T04:23:10.066191Z","iopub.execute_input":"2021-12-10T04:23:10.06663Z","iopub.status.idle":"2021-12-10T04:23:15.626485Z","shell.execute_reply.started":"2021-12-10T04:23:10.066588Z","shell.execute_reply":"2021-12-10T04:23:15.625504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_CNN_model():\n    model = Sequential(i)\n    model.add(layers.Conv2D(128, 5, strides=2, activation=\"relu\", input_shape=x_train[0].shape))\n    model.add(layers.Conv2D(128, 5, activation=\"relu\"))\n    model.add(layers.Conv2D(64, 3, activation=\"relu\"))  \n    model.add(layers.Conv2D(32, 3, activation=\"relu\"))    \n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dropout(0.5))\n    model.add(layers.Dense(5, activation='sigmoid'))\n    \n    model.compile(\n        loss='binary_crossentropy',\n        optimizer=Adam(lr=0.00005),\n        metrics=['accuracy']\n    )\n    \n    return model\n\ndef build_densenet_model():\n    model = Sequential()\n    model.add(densenet)\n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dropout(0.5))\n    model.add(layers.Dense(5, activation='sigmoid'))\n    \n    model.compile(\n        loss='binary_crossentropy',\n        optimizer=Adam(lr=0.00005),\n    )\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:23:15.659047Z","iopub.execute_input":"2021-12-10T04:23:15.659343Z","iopub.status.idle":"2021-12-10T04:23:15.677703Z","shell.execute_reply.started":"2021-12-10T04:23:15.659313Z","shell.execute_reply":"2021-12-10T04:23:15.676886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_cnn = build_CNN_model()\nmodel_cnn.build()\nmodel_cnn.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:23:16.485946Z","iopub.execute_input":"2021-12-10T04:23:16.486219Z","iopub.status.idle":"2021-12-10T04:23:16.55083Z","shell.execute_reply.started":"2021-12-10T04:23:16.486189Z","shell.execute_reply":"2021-12-10T04:23:16.550171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = model_cnn.fit_generator(\n#     data_generator,\n#     steps_per_epoch=x_train.shape[0] / BATCH_SIZE,\n#     epochs=10,\n#     validation_data=(x_val, y_val)\n# )","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:23:18.19445Z","iopub.execute_input":"2021-12-10T04:23:18.194696Z","iopub.status.idle":"2021-12-10T04:23:18.199971Z","shell.execute_reply.started":"2021-12-10T04:23:18.194668Z","shell.execute_reply":"2021-12-10T04:23:18.19904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with open('history.json', 'w') as f:\n#     json.dump(history.history, f)\n\n# history_df = pd.DataFrame(history.history)\n# history_df[['loss', 'val_loss']].plot()\n","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:23:18.513879Z","iopub.execute_input":"2021-12-10T04:23:18.514156Z","iopub.status.idle":"2021-12-10T04:23:18.518969Z","shell.execute_reply.started":"2021-12-10T04:23:18.514108Z","shell.execute_reply":"2021-12-10T04:23:18.517996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_densenet = build_densenet_model()\nmodel_densenet.build()\nmodel_densenet.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:23:19.260749Z","iopub.execute_input":"2021-12-10T04:23:19.261003Z","iopub.status.idle":"2021-12-10T04:23:20.03196Z","shell.execute_reply.started":"2021-12-10T04:23:19.260975Z","shell.execute_reply":"2021-12-10T04:23:20.03127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model_densenet.fit_generator(\n    data_generator,\n    steps_per_epoch=x_train.shape[0] / BATCH_SIZE,\n    epochs=30,\n    validation_data=(x_val, y_val)\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:23:20.033454Z","iopub.execute_input":"2021-12-10T04:23:20.034652Z","iopub.status.idle":"2021-12-10T04:42:11.440671Z","shell.execute_reply.started":"2021-12-10T04:23:20.034612Z","shell.execute_reply":"2021-12-10T04:42:11.439895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('history.json', 'w') as f:\n    json.dump(history.history, f)\n\nhistory_df = pd.DataFrame(history.history)\nhistory_df[['loss', 'val_loss']].plot()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:43:40.013384Z","iopub.execute_input":"2021-12-10T04:43:40.013653Z","iopub.status.idle":"2021-12-10T04:43:40.255728Z","shell.execute_reply.started":"2021-12-10T04:43:40.013625Z","shell.execute_reply":"2021-12-10T04:43:40.254962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = model_densenet.predict(x_test) > 0.5\ny_test = y_test.astype(int).sum(axis=1) - 1\n\ndf_test['diagnosis'] = y_test\ndf_test.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:43:46.410804Z","iopub.execute_input":"2021-12-10T04:43:46.411082Z","iopub.status.idle":"2021-12-10T04:43:51.83725Z","shell.execute_reply.started":"2021-12-10T04:43:46.411052Z","shell.execute_reply":"2021-12-10T04:43:51.836519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_df.plot()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:43:55.452653Z","iopub.execute_input":"2021-12-10T04:43:55.452979Z","iopub.status.idle":"2021-12-10T04:43:55.678315Z","shell.execute_reply.started":"2021-12-10T04:43:55.452876Z","shell.execute_reply":"2021-12-10T04:43:55.677645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_pred = model_densenet.predict(x_train) > 0.5\nx_train_pred = x_train_pred.astype(int).sum(axis=1) - 1\n\nx_val_pred = model_densenet.predict(x_val) > 0.5\nx_val_pred = x_val_pred.astype(int).sum(axis=1) - 1","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:44:12.550853Z","iopub.execute_input":"2021-12-10T04:44:12.551588Z","iopub.status.idle":"2021-12-10T04:44:24.028802Z","shell.execute_reply.started":"2021-12-10T04:44:12.55155Z","shell.execute_reply":"2021-12-10T04:44:24.028076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_pred","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:44:25.650749Z","iopub.execute_input":"2021-12-10T04:44:25.651764Z","iopub.status.idle":"2021-12-10T04:44:25.657623Z","shell.execute_reply.started":"2021-12-10T04:44:25.651716Z","shell.execute_reply":"2021-12-10T04:44:25.656796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true_train = np.sum(y_train, axis=1) - 1\ny_true_val = np.sum(y_val, axis=1) - 1","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:44:25.977555Z","iopub.execute_input":"2021-12-10T04:44:25.978059Z","iopub.status.idle":"2021-12-10T04:44:25.983339Z","shell.execute_reply.started":"2021-12-10T04:44:25.978022Z","shell.execute_reply":"2021-12-10T04:44:25.982573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, recall_score, precision_score, f1_score, accuracy_score\n\nprint(confusion_matrix(y_true_train, x_train_pred))\nprint(recall_score(y_true_train, x_train_pred, average='macro'))\nprint(precision_score(y_true_train, x_train_pred, average='macro'))\nprint(f1_score(y_true_train, x_train_pred, average='macro'))\nprint(accuracy_score(y_true_train, x_train_pred))","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:45:11.335653Z","iopub.execute_input":"2021-12-10T04:45:11.335922Z","iopub.status.idle":"2021-12-10T04:45:11.384544Z","shell.execute_reply.started":"2021-12-10T04:45:11.335881Z","shell.execute_reply":"2021-12-10T04:45:11.383783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(confusion_matrix(y_true_val, x_val_pred))\nprint(recall_score(y_true_val, x_val_pred, average='macro'))\nprint(precision_score(y_true_val, x_val_pred, average='macro'))\nprint(f1_score(y_true_val, x_val_pred, average='macro'))\nprint(accuracy_score(y_true_val, x_val_pred))\nprint(y_true_val.shape)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T04:45:13.307943Z","iopub.execute_input":"2021-12-10T04:45:13.308469Z","iopub.status.idle":"2021-12-10T04:45:13.32396Z","shell.execute_reply.started":"2021-12-10T04:45:13.308433Z","shell.execute_reply":"2021-12-10T04:45:13.323161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}