{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## SIIM Data Covid-19 ANN classifier\n#### Configuration","metadata":{}},{"cell_type":"code","source":"# default config\nmodel_name = \"DenseNet121\"\nlearning_rate = 0.001 # -------------------------- 0.001 - 0.0001\nmin_learning_rate = 1e-8\nbatch_size = 32# -------------------------------- 50 - 25\nepochs = 10# ------------------------------------ 10 - 20\nverbose = 1\nimg_process_function = \"equalize_adapthist\"\nisKaggleData = True\nclassification_type = \"multi\" #------------------ multi - binary\nclassifier = \"ann\"\n\n# for SVM classifier\ntrain_num = 5500 / batch_size # 5500 train - validation data\nval_num = 800 / batch_size   # 800 test data\nshow_cv_scores = False\nfeature_number = 128\n\nuse_fine_tuning = True #--------------------------- False - True\nuse_chex_weights = True\n\nlibraries = [\"pandas\",\"numpy\",\"sklearn\",\"tensorflow\",\"keras\",\"skimage\",\"matplotlib\",\"seaborn\"]\nshow_versions = True\n# svm_hyp_search = \"bayes\"\n\nshow_model_summary = False\nsave_weights = False\n\n# typical-none, atypical-none, indeterminate-none, all\nclasses = \"all\"","metadata":{"execution":{"iopub.status.busy":"2022-05-07T14:04:43.797562Z","iopub.execute_input":"2022-05-07T14:04:43.798253Z","iopub.status.idle":"2022-05-07T14:04:43.804939Z","shell.execute_reply.started":"2022-05-07T14:04:43.798215Z","shell.execute_reply":"2022-05-07T14:04:43.803751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score, classification_report, average_precision_score, f1_score, precision_score, recall_score\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.svm import SVC\nfrom skimage import exposure\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Input, Dense, Flatten, Conv2D, MaxPooling2D, Dropout, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras import models\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint,  EarlyStopping\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential\n\nfrom tensorflow.keras.metrics import Recall,Precision\n\nfrom tensorflow.keras.layers import InputLayer, BatchNormalization, Dropout, Flatten, Dense, Activation, MaxPool2D, Conv2D\n\nfrom tensorflow.keras.applications import InceptionV3, DenseNet121, ResNet50, Xception\n\nimport importlib\nfrom skimage import exposure\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-05-07T14:04:43.875781Z","iopub.execute_input":"2022-05-07T14:04:43.875983Z","iopub.status.idle":"2022-05-07T14:04:43.886866Z","shell.execute_reply.started":"2022-05-07T14:04:43.87596Z","shell.execute_reply":"2022-05-07T14:04:43.886126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_versions(libraries = None):\n    \n    from importlib import import_module\n    \n    for library in libraries:\n        print(f\"{library} version: {import_module(library).__version__}\")\n\nif show_versions:\n    display_versions(libraries)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T14:04:43.924866Z","iopub.execute_input":"2022-05-07T14:04:43.925055Z","iopub.status.idle":"2022-05-07T14:04:43.931599Z","shell.execute_reply.started":"2022-05-07T14:04:43.925033Z","shell.execute_reply":"2022-05-07T14:04:43.930712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Transfer Models","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications import VGG16, VGG19, InceptionV3, NASNetMobile, NASNetLarge, DenseNet121, ResNet50, Xception, InceptionResNetV2, EfficientNetB7\n\n\ndef get_models():\n    \n    models_ = dict(\n                   # this is used for ChexNet\n                    DenseNet121=dict(\n                        input_shape=(224, 224, 3),\n                        module_name=\"densenet\",\n                        last_conv_layer=\"conv5_block16_concat\",\n                    ),\n                    ResNet50=dict(\n                        input_shape=(224, 224, 3),\n                        module_name=\"resnet\",\n                        last_conv_layer=\"conv5_block3_out\",\n                    ),\n                    InceptionV3=dict(\n                        input_shape=(299, 299, 3),\n                        module_name=\"inception_v3\",\n                        last_conv_layer=\"mixed10\",\n                    ),\n                    Xception=dict(\n                        input_shape=(299, 299, 3),\n                        module_name=\"xception\",\n                        last_conv_layer=\"block14_sepconv2_act\",\n                    ),\n                )\n    \n    return models_","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-05-07T14:04:43.975191Z","iopub.execute_input":"2022-05-07T14:04:43.975481Z","iopub.status.idle":"2022-05-07T14:04:43.984684Z","shell.execute_reply.started":"2022-05-07T14:04:43.975454Z","shell.execute_reply":"2022-05-07T14:04:43.983927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_data_for_kaggle():\n    \n    df_image = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\n    df_study = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\n    df_study['id'] = df_study['id'].str.replace('_study',\"\")\n    df_study.rename({'id': 'StudyInstanceUID'},axis=1, inplace=True)\n    df_train = df_image.merge(df_study, on='StudyInstanceUID')\n    df_train.loc[df_train['Negative for Pneumonia']==1, 'study_label'] = 'negative'\n    df_train.loc[df_train['Typical Appearance']==1, 'study_label'] = 'typical'\n    df_train.loc[df_train['Indeterminate Appearance']==1, 'study_label'] = 'indeterminate'\n    df_train.loc[df_train['Atypical Appearance']==1, 'study_label'] = 'atypical'\n    df_train.drop(['Negative for Pneumonia','Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance'], axis=1, inplace=True)\n    df_train['id'] = df_train['id'].str.replace('_image', '.jpg')\n    df_train['image_label'] = df_train['label'].str.split().apply(lambda x : x[0])\n    df_size = pd.read_csv('../input/covid-jpg-512/size.csv')\n    data = df_train.merge(df_size, on='id')\n    data = data.drop([\"boxes\",\"label\",\"StudyInstanceUID\",\"dim0\",\"dim1\",\"split\"], axis = 1)\n    img_dir = \"../input/covid-jpg-512/train\"\n    \n    return data, img_dir\n\nif isKaggleData:\n    data, img_dir = prepare_data_for_kaggle()\nelse:\n    data = pd.read_csv(\"../train_data.csv\")\n    img_dir = \"../images/train\"\n    \ndf_data = data.copy()","metadata":{"execution":{"iopub.status.busy":"2022-05-07T14:04:44.027382Z","iopub.execute_input":"2022-05-07T14:04:44.027797Z","iopub.status.idle":"2022-05-07T14:04:44.112189Z","shell.execute_reply.started":"2022-05-07T14:04:44.027769Z","shell.execute_reply":"2022-05-07T14:04:44.111509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# drop images from dataframe not in images directory\nfiles = os.listdir(\"../input/covid-jpg-512/train\")\n\nnot_in_files_index = []\n\nfor file_id in df_data.id:\n    if file_id in files:\n        continue\n    else:\n        not_in_files_index.append(df_data[df_data[\"id\"] == file_id].index[0])\n        \ndf_data = df_data.drop(not_in_files_index, axis = 0)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T14:04:44.113643Z","iopub.execute_input":"2022-05-07T14:04:44.113888Z","iopub.status.idle":"2022-05-07T14:04:44.533376Z","shell.execute_reply.started":"2022-05-07T14:04:44.113855Z","shell.execute_reply":"2022-05-07T14:04:44.53265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop images that have unclear view\ndrop_df = pd.read_csv(\"../input/dropped-siim/dropped_image_IDs.csv\") + \".jpg\"\n# splitting images train and test\n\ndrop_index = []\nfor row in drop_df.values:\n    drop_index.append(df_data[df_data[\"id\"] == row[0]].index[0])\n            \ndf_data = df_data.drop(drop_index, axis = 0)\n\n# other binary classification\nif classes == \"typical-none\":\n    df_data = df_data.drop(df_data[(df_data[\"study_label\"] == \"atypical\") | (df_data[\"study_label\"] == \"indeterminate\")].index, axis = 0)\n    df_train = df_data.iloc[:int(len(df_data) * 0.80)]\n    df_test = df_data.iloc[int(len(df_data) * 0.80):]\nelif classes == \"atypical-none\":\n    df_data = df_data.drop(df_data[(df_data[\"study_label\"] == \"typical\") | (df_data[\"study_label\"] == \"indeterminate\")].index, axis = 0)\n    df_train = df_data.iloc[:int(len(df_data) * 0.80)]\n    df_test = df_data.iloc[int(len(df_data) * 0.80):]\nelif classes == \"indeterminate-none\":\n    df_data = df_data.drop(df_data[(df_data[\"study_label\"] == \"typical\") | (df_data[\"study_label\"] == \"atypical\")].index, axis = 0)\n    df_train = df_data.iloc[:int(len(df_data) * 0.85)]\n    df_test = df_data.iloc[int(len(df_data) * 0.85):]\nelse:    \n    df_train = df_data.iloc[:5500]\n    df_test = df_data.iloc[5500:]","metadata":{"execution":{"iopub.status.busy":"2022-05-07T14:04:44.535075Z","iopub.execute_input":"2022-05-07T14:04:44.535357Z","iopub.status.idle":"2022-05-07T14:04:44.585297Z","shell.execute_reply.started":"2022-05-07T14:04:44.535309Z","shell.execute_reply":"2022-05-07T14:04:44.584682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model Building","metadata":{}},{"cell_type":"code","source":"models_ = get_models()\n\nfor model_name in models_.keys():\n    \n    input_shape = models_[model_name][\"input_shape\"]\n    img_size = input_shape[0]\n    \n    def generate_images_for_model_training(classifier, classification_type, img_process_function, df_train, df_test, img_dir, img_size, batch_size, validation_split = 0.20):\n    \n        from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n        from skimage import exposure\n\n        # Defined image preprocessing functions\n\n        def preprocess_function(img):\n\n            if img_process_function == \"equalize_adapthist\":\n                img = exposure.equalize_adapthist(img/255, clip_limit=0.03, kernel_size=24)\n            elif img_process_function == \"equalize_hist\":\n                img = exposure.equalize_hist(img/255, clip_limit=0.03, kernel_size=24)\n            elif img_process_function == \"rescale_intensity\":\n                img = exposure.rescale_intensity(img/255, clip_limit=0.03, kernel_size=24)\n\n            return img\n\n        if classification_type == \"binary\":\n            y_col = \"image_label\"\n        else:\n            y_col = \"study_label\"\n\n\n        image_generator_train = ImageDataGenerator(\n                        featurewise_center=False,\n                        samplewise_center=False,\n                        featurewise_std_normalization=False,\n                        samplewise_std_normalization=False,\n                        zca_epsilon=1e-06,\n                        zca_whitening=False,\n                        width_shift_range=0.0,\n                        height_shift_range=0.0,\n                        brightness_range=[0.8, 1.1],\n                        shear_range=0.1,\n                        zoom_range=0.1,\n                        channel_shift_range=0.0,\n                        cval=0.0,\n                        horizontal_flip=False,\n                        vertical_flip=False,\n                        rescale=None,\n                        rotation_range=30,\n                        preprocessing_function=preprocess_function,\n                        validation_split=validation_split)\n\n        image_generator_valid = ImageDataGenerator(validation_split=validation_split,\n                                                   preprocessing_function=preprocess_function)\n\n\n        train_generator = image_generator_train.flow_from_dataframe(\n                    dataframe = df_train,\n                    directory=img_dir,\n                    x_col = 'id',\n                    y_col =  y_col,  \n                    target_size=(img_size, img_size),\n                    batch_size=batch_size,\n                    subset='training', \n                    seed = 42, \n                    class_mode = \"categorical\") \n\n        valid_generator = image_generator_valid.flow_from_dataframe(\n                dataframe = df_train,\n                directory=img_dir,\n                x_col = 'id',\n                y_col = y_col,\n                target_size=(img_size, img_size),\n                batch_size=batch_size,\n                subset='validation', \n                shuffle=False,  \n                seed=42, \n                class_mode = \"categorical\")\n\n        return train_generator, valid_generator\n\n    train_generator, valid_generator = generate_images_for_model_training( classifier = classifier, \n                                                                           classification_type = classification_type, \n                                                                           img_process_function = img_process_function, \n                                                                           df_train = df_train, \n                                                                           df_test = df_test, \n                                                                           img_dir = img_dir, \n                                                                           img_size = img_size, \n                                                                           batch_size = batch_size, \n                                                                           validation_split = 0.15)\n    \n    \n    def get_last_conv_layer(base_model, model_name):\n    \n        models_ = get_models()\n        layer = base_model.get_layer(models_[model_name][\"last_conv_layer\"])\n\n        return layer\n    \n\n    base_model_class = getattr(\n        importlib.import_module(\n            f\"keras.applications.{models_[model_name]['module_name']}\"\n            ),\n            model_name)\n\n    img_input = Input(shape = input_shape)\n\n    base_model = base_model_class(\n                include_top = False,\n                input_tensor = img_input,\n                input_shape = input_shape,\n                weights = \"imagenet\",\n                pooling = \"avg\")\n    \n\n    if (model_name == \"DenseNet121\") & use_chex_weights:\n\n        chex_weights_path = '../input/chexnet-weights/brucechou1983_CheXNet_Keras_0.3.0_weights.h5'\n        out = Dense(14, activation='sigmoid')(base_model.output)\n        base_model = Model(inputs=base_model.input, outputs=out)\n        base_model.load_weights(chex_weights_path)\n        base_model.trainable = False\n        x = get_last_conv_layer(base_model, model_name).output\n        output = GlobalAveragePooling2D()(x)\n        output = Dropout(0.1)(output)\n\n    else:\n        base_model.trainable = False\n        x = get_last_conv_layer(base_model, model_name).output\n        output = GlobalAveragePooling2D()(x)\n        output = Dropout(0.1)(output)\n\n    if use_fine_tuning:\n\n        base_model.trainable = True\n\n        if classification_type == \"multi\":\n            predictions = Dense(len(df_train.study_label.unique()), activation = \"softmax\", name = \"multi_predictions\")(output)\n            model = Model(base_model.input, predictions)\n            model.compile(Adam(lr=learning_rate),loss='categorical_crossentropy',metrics=['accuracy',Precision(),Recall()])\n\n        else:\n            predictions = Dense(len(df_train.image_label.unique()), activation = \"softmax\", name = \"binary_predictions\")(output)\n            model = Model(base_model.input, predictions)\n            model.compile(Adam(lr=learning_rate),loss='binary_crossentropy',metrics=['accuracy',Precision(),Recall()])\n\n        if show_model_summary:\n            print(model.summary())\n\n        # Keras callbacks\n        rlr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.1, patience = 2, verbose = verbose, \n                                        min_delta = 1e-4, min_lr = min_learning_rate, mode = 'min')\n\n        es = EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, patience = 5, mode = 'min', \n                            restore_best_weights = True, verbose = verbose)\n\n        ckp = ModelCheckpoint('model.h5',monitor = 'val_loss',\n                              verbose = verbose, save_best_only = True, mode = 'min')\n        \n        print(\"Training and Validation........................\")\n\n        # Model fitting\n        history = model.fit(\n              train_generator,\n              epochs= epochs,\n              validation_data=valid_generator,\n              callbacks=[es, rlr, ckp],\n              verbose= verbose\n              )\n\n        if save_weights:\n            model.save_weights(f\"{model_name}-model.h5\")\n\n    else:\n        if classification_type == \"multi\":\n\n            # Building Connecting Model\n            predictions = Dense(len(df_train.study_label.unique()), activation = \"softmax\", name = \"multi_predictions\")(output)\n            model = Model(base_model.input, predictions)\n            model.compile(Adam(lr=learning_rate),loss='categorical_crossentropy',metrics=['accuracy',Precision(),Recall()])\n\n        else:\n            predictions = Dense(len(df_train.image_label.unique()), activation = \"softmax\", name = \"binary_predictions\")(output)\n            model = Model(base_model.input, predictions)\n            model.compile(Adam(lr=learning_rate),loss='categorical_crossentropy',metrics=['accuracy',\"AUC\",Precision(),Recall()])\n\n        # Keras callbacks\n        rlr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.1, patience = 2, verbose = verbose, \n                                            min_delta = 1e-4, min_lr = min_learning_rate, mode = 'min')\n\n        es = EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, patience = 5, mode = 'min', \n                                restore_best_weights = True, verbose = verbose)\n\n        ckp = ModelCheckpoint('model.h5',monitor = 'val_loss',\n                                  verbose = verbose, save_best_only = True, mode = 'min')\n        \n        print(\"Training and Validation........................\")\n\n        # Model fitting\n        history = model.fit(\n                  train_generator,\n                  epochs= epochs,\n                  validation_data=valid_generator,\n                  callbacks=[es, rlr, ckp],\n                  verbose= verbose\n                )\n\n\n            \n    def plot_tl_metrics(history, model_name):\n\n        hist = pd.DataFrame(history.history)\n        hist.index += 1\n\n        fig, (ax1, ax2) = plt.subplots(figsize=(12,12),nrows=2, ncols=1)\n        hist['loss'].plot(ax=ax1,c='k',label='Eğitim')\n        hist['val_loss'].plot(ax=ax1,c='r',linestyle='--', label='Doğrulama')\n        ax1.legend()\n\n        hist['accuracy'].plot(ax=ax2,c='k',label='Eğitim')\n        hist['val_accuracy'].plot(ax=ax2,c='r',linestyle='--',label='Doğrulama')\n        ax2.legend()\n        plt.suptitle(f\"{model_name} Kayıp ve Doğruluk Grafikleri\")\n        plt.show()\n        \n        \n    plot_tl_metrics(history, model_name)\n    \n    print(\"Testing.................\")\n    \n    def generate_test_images(classifier, classification_type, \n                       img_process_function, df_train, df_test, img_dir, \n                       img_size, batch_size, validation_split = 0.20):\n\n        from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n        from skimage import exposure\n\n        if classification_type == \"binary\":\n            y_col = \"image_label\"\n        else:\n            y_col = \"study_label\"\n\n        def preprocess_function(img):\n\n            if img_process_function == \"equalize_adapthist\":\n                img = exposure.equalize_adapthist(img/255, clip_limit=0.03, kernel_size=24)\n            elif img_process_function == \"equalize_hist\":\n                img = exposure.equalize_hist(img/255, clip_limit=0.03, kernel_size=24)\n            elif img_process_function == \"rescale_intensity\":\n                img = exposure.rescale_intensity(img/255, clip_limit=0.03, kernel_size=24)\n\n            return img\n\n\n        image_generator_test = ImageDataGenerator(preprocessing_function=preprocess_function)\n\n        test_generator = image_generator_test.flow_from_dataframe(\n                    dataframe = df_test,\n                    directory=img_dir,\n                    x_col = 'id',\n                    y_col = y_col,\n                    target_size=(img_size, img_size),\n                    batch_size=batch_size,\n                    shuffle=False,  \n                    seed=42, \n                    class_mode = \"categorical\")\n\n        return test_generator\n\n    test_generator = generate_test_images(classifier, \n                                         classification_type, \n                                         img_process_function, \n                                         df_train, df_test, \n                                         img_dir, img_size, batch_size)\n    \n\n    \n    actual =  test_generator.labels\n    preds_ = model.predict(test_generator)\n    preds = np.argmax(model.predict(test_generator), axis=1)\n    cfmx = confusion_matrix(actual, preds)\n\n    sns.heatmap(cfmx, annot=True, cmap='Blues',\n        xticklabels=list(test_generator.class_indices.keys()),\n            fmt='.0f', \n            yticklabels=list(test_generator.class_indices.keys())\n            )\n\n    plt.xlabel(\"Predictions\")\n    plt.ylabel(\"True Labels\")\n    plt.show()\n    \n    print(\"Results:\")\n    print(\"-------------------------\")\n    print(\"Model name:\",model_name)\n    print(\"Classification type:\",classification_type)\n    print(\"Classifier:\",classifier)\n    print(\"Image preprocess function:\",img_process_function)\n\n    print(\"-------------------------\")\n    print(\"Batch size:\",batch_size)\n    print(\"Learning rate:\",learning_rate)\n    print(\"Number of Epoch:\",epochs)\n\n    if model_name == \"DenseNet121\":\n        print(\"Chexnet weights status:\",use_chex_weights)\n\n    print(\"Classification Metrics:\")\n    print(\"-------------------------\")\n    \n    print(classification_report(actual, preds, digits = 3))\n    \n    if classification_type == \"binary\":\n        def acc_score_manual(cfmx):\n            TP = cfmx[1][1]\n            FP = cfmx[1][0]\n            FN = cfmx[0][1]\n            TN = cfmx[0][0]\n            return (TP + TN) / (TP + FN + FP + TN)\n\n        def spe_score_manual(cfmx):\n            TP = cfmx[1][1]\n            FP = cfmx[1][0]\n            FN = cfmx[0][1]\n            TN = cfmx[0][0]\n            return (TN) / (FP + TN)\n\n        def sen_rec_score_manual(cfmx):\n            TP = cfmx[1][1]\n            FP = cfmx[1][0]\n            FN = cfmx[0][1]\n            TN = cfmx[0][0]\n            return (TP) / (TP + FN)\n\n        def pre_score_manual(cfmx):\n            TP = cfmx[1][1]\n            FP = cfmx[1][0]\n            FN = cfmx[0][1]\n            TN = cfmx[0][0]\n            return (TP) / (TP + FP)\n\n        def f1_score_manual(cfmx):\n            TP = cfmx[1][1]\n            FP = cfmx[1][0]\n            FN = cfmx[0][1]\n            TN = cfmx[0][0]\n            return (2*TP) / (FP + FN + (2*TP))\n\n        acc = acc_score_manual(cfmx)\n        spe = spe_score_manual(cfmx)\n        sen = sen_rec_score_manual(cfmx)\n        pre = pre_score_manual(cfmx)\n        f11 = f1_score_manual(cfmx)\n        \n        print(\"Accuracy: \",acc)\n        print(\"Specificity:\",spe)\n        print(\"Precision:\",pre)\n        print(\"Sensitivity *Recall*:\",sen)\n        print(\"F1 score:\",f11)\n        print(\"ROC AUC Score\",roc)\n        \n    print(\"Accuracy: \",accuracy_score)\n    print(\"Weighted Precision:\",precision_score(actual, preds, average = \"weighted\"))\n    print(\"Weighted Sensitivity *Recall*:\",recall_score(actual, preds, average = \"weighted\"))\n    print(\"Weighted F1 score:\",f1_score(actual, preds, average = \"weighted\"))\n    if classification_type == \"binary\":\n        print(\"Weighted ROC AUC Score\",roc_auc_score(actual, preds, average=\"weighted\"))\n    else:\n        print(\"Weighted ROC AUC Score\",roc_auc_score(actual, preds_, average=\"weighted\", multi_class=\"ovr\"))","metadata":{"execution":{"iopub.status.busy":"2022-05-07T14:04:44.625443Z","iopub.execute_input":"2022-05-07T14:04:44.62582Z","iopub.status.idle":"2022-05-07T17:37:07.096826Z","shell.execute_reply.started":"2022-05-07T14:04:44.625776Z","shell.execute_reply":"2022-05-07T17:37:07.096064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}