{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## SIIM Data Covid-19 SVM classifier\n#### Configuration","metadata":{}},{"cell_type":"code","source":"# default config\nmodel_name = \"DenseNet121\"\nlearning_rate = 0.0001 # -------------------------- 0.001 - 0.0001\nmin_learning_rate = 1e-8\nbatch_size = 25 # -------------------------------- 50 - 25\nepochs = 5 # ------------------------------------ 10 - 20\nverbose = 1\nimg_process_function = \"equalize_adapthist\"\nisKaggleData = True\nclassification_type = \"binary\" #------------------ multi - binary\nclassifier = \"svm\"\n\ntrain_num = int(500 / batch_size)\nval_num = int(150 / batch_size)\nshow_cv_scores = False\nfeature_number = 1000\n\nuse_fine_tuning = True #--------------------------- False - True\nuse_chex_weights = True\n\nlibraries = [\"pandas\",\"numpy\",\"sklearn\",\"tensorflow\",\"keras\",\"skimage\",\"matplotlib\",\"seaborn\"]\nshow_versions = True\nsvm_hyp_search = \"bayes\"\n\nshow_model_summary = False\nsave_weights = False\n\n# typical-none, atypical-none, indeterminate-none, all\nclasses = \"indeterminate-none\"","metadata":{"execution":{"iopub.status.busy":"2022-05-05T12:58:58.547766Z","iopub.execute_input":"2022-05-05T12:58:58.548067Z","iopub.status.idle":"2022-05-05T12:58:58.555108Z","shell.execute_reply.started":"2022-05-05T12:58:58.548031Z","shell.execute_reply":"2022-05-05T12:58:58.554133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score, classification_report, average_precision_score, f1_score, precision_score, recall_score\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.svm import SVC\nfrom skimage import exposure\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import Input, Dense, Flatten, Conv2D, MaxPooling2D, Dropout, GlobalAveragePooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras import models\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint,  EarlyStopping\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential\n\nfrom tensorflow.keras.metrics import Recall,Precision\n\nfrom tensorflow.keras.layers import InputLayer, BatchNormalization, Dropout, Flatten, Dense, Activation, MaxPool2D, Conv2D\n\nfrom tensorflow.keras.applications import InceptionV3, DenseNet121, ResNet50, Xception\n\nimport importlib\nfrom skimage import exposure\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-05-05T12:58:58.58627Z","iopub.execute_input":"2022-05-05T12:58:58.587118Z","iopub.status.idle":"2022-05-05T12:58:58.595391Z","shell.execute_reply.started":"2022-05-05T12:58:58.587074Z","shell.execute_reply":"2022-05-05T12:58:58.59449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_versions(libraries = None):\n    \n    from importlib import import_module\n    \n    for library in libraries:\n        print(f\"{library} version: {import_module(library).__version__}\")\n\nif show_versions:\n    display_versions(libraries)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T12:58:58.607477Z","iopub.execute_input":"2022-05-05T12:58:58.607942Z","iopub.status.idle":"2022-05-05T12:58:58.61683Z","shell.execute_reply.started":"2022-05-05T12:58:58.607907Z","shell.execute_reply":"2022-05-05T12:58:58.616039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Transfer Models","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications import InceptionV3, DenseNet121, ResNet50, Xception\n\n\ndef get_models():\n    \n    models_ = dict(\n                    \n                    # this is used for ChexNet\n                    DenseNet121=dict(\n                        input_shape=(224, 224, 3),\n                        module_name=\"densenet\",\n                        last_conv_layer=\"conv5_block16_concat\",\n                    ),\n                    ResNet50=dict(\n                        input_shape=(224, 224, 3),\n                        module_name=\"resnet\",\n                        last_conv_layer=\"conv5_block3_out\",\n                    ),\n                    InceptionV3=dict(\n                        input_shape=(299, 299, 3),\n                        module_name=\"inception_v3\",\n                        last_conv_layer=\"mixed10\",\n                    )\n                    Xception=dict(\n                        input_shape=(299, 299, 3),\n                        module_name=\"xception\",\n                        last_conv_layer=\"block14_sepconv2_act\",\n                    )\n                \n                )\n    \n    return models_","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-05-05T12:58:58.633261Z","iopub.execute_input":"2022-05-05T12:58:58.633535Z","iopub.status.idle":"2022-05-05T12:58:58.644135Z","shell.execute_reply.started":"2022-05-05T12:58:58.633506Z","shell.execute_reply":"2022-05-05T12:58:58.643166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_ = get_models()\ninput_shape = models_[model_name][\"input_shape\"]\nimg_size = input_shape[0]\n\ndef prepare_data_for_kaggle():\n    \n    df_image = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')\n    df_study = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\n    df_study['id'] = df_study['id'].str.replace('_study',\"\")\n    df_study.rename({'id': 'StudyInstanceUID'},axis=1, inplace=True)\n    df_train = df_image.merge(df_study, on='StudyInstanceUID')\n    df_train.loc[df_train['Negative for Pneumonia']==1, 'study_label'] = 'negative'\n    df_train.loc[df_train['Typical Appearance']==1, 'study_label'] = 'typical'\n    df_train.loc[df_train['Indeterminate Appearance']==1, 'study_label'] = 'indeterminate'\n    df_train.loc[df_train['Atypical Appearance']==1, 'study_label'] = 'atypical'\n    df_train.drop(['Negative for Pneumonia','Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance'], axis=1, inplace=True)\n    df_train['id'] = df_train['id'].str.replace('_image', '.jpg')\n    df_train['image_label'] = df_train['label'].str.split().apply(lambda x : x[0])\n    df_size = pd.read_csv('../input/covid-jpg-512/size.csv')\n    data = df_train.merge(df_size, on='id')\n    data = data.drop([\"boxes\",\"label\",\"StudyInstanceUID\",\"dim0\",\"dim1\",\"split\"], axis = 1)\n    img_dir = \"../input/covid-jpg-512/train\"\n    \n    return data, img_dir\n\nif isKaggleData:\n    data, img_dir = prepare_data_for_kaggle()\nelse:\n    data = pd.read_csv(\"../train_data.csv\")\n    img_dir = \"../images/train\"\n    \ndf_data = data.copy()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T12:58:58.652732Z","iopub.execute_input":"2022-05-05T12:58:58.653205Z","iopub.status.idle":"2022-05-05T12:58:58.748127Z","shell.execute_reply.started":"2022-05-05T12:58:58.653145Z","shell.execute_reply":"2022-05-05T12:58:58.746927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# drop images from dataframe not in images directory\nfiles = os.listdir(\"../input/covid-jpg-512/train\")\n\nnot_in_files_index = []\n\nfor file_id in df_data.id:\n    if file_id in files:\n        continue\n    else:\n        not_in_files_index.append(df_data[df_data[\"id\"] == file_id].index[0])\n        \ndf_data = df_data.drop(not_in_files_index, axis = 0)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T12:58:58.79972Z","iopub.execute_input":"2022-05-05T12:58:58.800003Z","iopub.status.idle":"2022-05-05T12:58:59.115122Z","shell.execute_reply.started":"2022-05-05T12:58:58.799974Z","shell.execute_reply":"2022-05-05T12:58:59.114032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop images that have unclear view\ndrop_df = pd.read_csv(\"../input/dropped-siim/dropped_image_IDs.csv\") + \".jpg\"\n# splitting images train and test\n\ndrop_index = []\nfor row in drop_df.values:\n    drop_index.append(df_data[df_data[\"id\"] == row[0]].index[0])\n            \ndf_data = df_data.drop(drop_index, axis = 0)\n\n# other binary classification\nif classes == \"typical-none\":\n    df_data = df_data.drop(df_data[(df_data[\"study_label\"] == \"atypical\") | (df_data[\"study_label\"] == \"indeterminate\")].index, axis = 0)\n    df_train = df_data.iloc[:int(len(df_data) * 0.80)]\n    df_test = df_data.iloc[int(len(df_data) * 0.80):]\nelif classes == \"atypical-none\":\n    df_data = df_data.drop(df_data[(df_data[\"study_label\"] == \"typical\") | (df_data[\"study_label\"] == \"indeterminate\")].index, axis = 0)\n    df_train = df_data.iloc[:int(len(df_data) * 0.80)]\n    df_test = df_data.iloc[int(len(df_data) * 0.80):]\nelif classes == \"indeterminate-none\":\n    df_data = df_data.drop(df_data[(df_data[\"study_label\"] == \"typical\") | (df_data[\"study_label\"] == \"atypical\")].index, axis = 0)\n    df_train = df_data.iloc[:int(len(df_data) * 0.85)]\n    df_test = df_data.iloc[int(len(df_data) * 0.85):]\nelse:    \n    df_train = df_data.iloc[:5500]\n    df_test = df_data.iloc[5500:]","metadata":{"execution":{"iopub.status.busy":"2022-05-05T12:58:59.117294Z","iopub.execute_input":"2022-05-05T12:58:59.117653Z","iopub.status.idle":"2022-05-05T12:58:59.165437Z","shell.execute_reply.started":"2022-05-05T12:58:59.117609Z","shell.execute_reply":"2022-05-05T12:58:59.16434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Image Generators for Training","metadata":{}},{"cell_type":"code","source":"def generate_images_for_model_training(classifier, classification_type, img_process_function, df_train, df_test, img_dir, img_size, batch_size, validation_split = 0.15):\n    \n    from tensorflow.keras.preprocessing.image import ImageDataGenerator\n    \n    from skimage import exposure\n    \n    # Defined image preprocessing functions\n\n    def preprocess_function(img):\n        \n        if img_process_function == \"equalize_adapthist\":\n            img = exposure.equalize_adapthist(img/255, clip_limit=0.03, kernel_size=24)\n        elif img_process_function == \"equalize_hist\":\n            img = exposure.equalize_hist(img/255, clip_limit=0.03, kernel_size=24)\n        elif img_process_function == \"rescale_intensity\":\n            img = exposure.rescale_intensity(img/255, clip_limit=0.03, kernel_size=24)\n            \n        return img\n    \n    if classification_type == \"binary\":\n        y_col = \"image_label\"\n    else:\n        y_col = \"study_label\"\n    \n    \n    image_generator_train = ImageDataGenerator(\n                    featurewise_center=False,\n                    samplewise_center=False,\n                    featurewise_std_normalization=False,\n                    samplewise_std_normalization=False,\n                    zca_epsilon=1e-06,\n                    zca_whitening=False,\n                    width_shift_range=0.0,\n                    height_shift_range=0.0,\n                    brightness_range=[0.8, 1.1],\n                    shear_range=0.1,\n                    zoom_range=0.0,\n                    channel_shift_range=0.0,\n                    cval=0.0,\n                    horizontal_flip=False,\n                    vertical_flip=False,\n                    rescale=None,\n                    rotation_range=20,\n                    preprocessing_function=preprocess_function,\n                    validation_split=validation_split)\n        \n    image_generator_valid = ImageDataGenerator(validation_split=validation_split,\n                                               preprocessing_function=preprocess_function)\n      \n\n    train_generator = image_generator_train.flow_from_dataframe(\n                dataframe = df_train,\n                directory=img_dir,\n                x_col = 'id',\n                y_col =  y_col,  \n                target_size=(img_size, img_size),\n                batch_size=batch_size,\n                subset='training', \n                seed = 42, \n                class_mode = \"categorical\") \n        \n    valid_generator = image_generator_valid.flow_from_dataframe(\n            dataframe = df_train,\n            directory=img_dir,\n            x_col = 'id',\n            y_col = y_col,\n            target_size=(img_size, img_size),\n            batch_size=batch_size,\n            subset='validation', \n            shuffle=False,  \n            seed=42, \n            class_mode = \"categorical\")\n    \n    return train_generator, valid_generator\n\ntrain_generator, valid_generator = generate_images_for_model_training( classifier = classifier, \n                                                                       classification_type = classification_type, \n                                                                       img_process_function = img_process_function, \n                                                                       df_train = df_train, \n                                                                       df_test = df_test, \n                                                                       img_dir = img_dir, \n                                                                       img_size = img_size, \n                                                                       batch_size = batch_size, \n                                                                       validation_split = 0.15)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-05-05T12:58:59.167468Z","iopub.execute_input":"2022-05-05T12:58:59.167864Z","iopub.status.idle":"2022-05-05T12:59:00.486179Z","shell.execute_reply.started":"2022-05-05T12:58:59.167813Z","shell.execute_reply":"2022-05-05T12:59:00.485509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model Building","metadata":{}},{"cell_type":"code","source":"def get_last_conv_layer(base_model, model_name):\n    \n    models_ = get_models()\n    layer = base_model.get_layer(models_[model_name][\"last_conv_layer\"])\n    \n    return layer\n\nbase_model_class = getattr(\n    importlib.import_module(\n        f\"keras.applications.{models_[model_name]['module_name']}\"\n        ),\n        model_name)\n          \nimg_input = Input(shape = input_shape)\n        \nbase_model = base_model_class(\n            include_top = False,\n            input_tensor = img_input,\n            input_shape = input_shape,\n            weights = \"imagenet\",\n            pooling = \"avg\")\n\nif (model_name == \"DenseNet121\") & use_chex_weights:\n    \n    chex_weights_path = '../input/chexnet-weights/brucechou1983_CheXNet_Keras_0.3.0_weights.h5'\n    out = Dense(14, activation='sigmoid')(base_model.output)\n    base_model = Model(inputs=base_model.input, outputs=out)\n    base_model.load_weights(chex_weights_path)\n    x = get_last_conv_layer(base_model, model_name).output\n    output = GlobalAveragePooling2D()(x)\n    \nelse:\n    x = get_last_conv_layer(base_model, model_name).output\n    output = GlobalAveragePooling2D()(x)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T12:59:00.487268Z","iopub.execute_input":"2022-05-05T12:59:00.487616Z","iopub.status.idle":"2022-05-05T12:59:04.904784Z","shell.execute_reply.started":"2022-05-05T12:59:00.487587Z","shell.execute_reply":"2022-05-05T12:59:04.90381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Model Training and Evaluation","metadata":{"tags":[]}},{"cell_type":"code","source":"base_model.trainable = False\n\nif use_fine_tuning:   \n    base_model.trainable = True\n    \n    if classification_type == \"multi\":\n        predictions = Dense(len(df_train.study_label.unique()), activation = \"softmax\", name = \"multi_predictions\")(output)\n        model = Model(base_model.input, predictions)\n        model.compile(Adam(lr=learning_rate),loss='categorical_crossentropy',metrics=['accuracy'])\n        \n    else:\n        predictions = Dense(len(df_train.image_label.unique()), activation = \"softmax\", name = \"binary_predictions\")(output)\n        model = Model(base_model.input, predictions)\n        model.compile(Adam(lr=learning_rate),loss='binary_crossentropy',metrics=['accuracy'])\n        \n    if show_model_summary:\n        print(model.summary())\n        \n    # Keras callbacks\n    rlr = ReduceLROnPlateau(monitor = 'val_loss', factor = 0.1, patience = 2, verbose = verbose, \n                                    min_delta = 1e-4, min_lr = min_learning_rate, mode = 'min')\n    \n    es = EarlyStopping(monitor = 'val_loss', min_delta = 1e-4, patience = 5, mode = 'min', \n                        restore_best_weights = True, verbose = verbose)\n    \n    ckp = ModelCheckpoint('model.h5',monitor = 'val_loss',\n                          verbose = verbose, save_best_only = True, mode = 'min')\n    \n    # Model fitting\n    history = model.fit(\n          train_generator,\n          epochs= epochs,\n          validation_data=valid_generator,\n          callbacks=[es, rlr, ckp],\n          verbose= verbose\n          )\n    \n    if save_weights:\n        model.save_weights(f\"{model_name}-model.h5\")\n    ","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-05-05T12:59:04.907387Z","iopub.execute_input":"2022-05-05T12:59:04.907756Z","iopub.status.idle":"2022-05-05T13:41:50.051278Z","shell.execute_reply.started":"2022-05-05T12:59:04.90771Z","shell.execute_reply":"2022-05-05T13:41:50.050666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Feature Layer for SVM Classifier","metadata":{}},{"cell_type":"code","source":"output = Dense(feature_number, activation=\"relu\", name = \"features\")(output)\nmodel = Model(base_model.input, output)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:41:50.052608Z","iopub.execute_input":"2022-05-05T13:41:50.052954Z","iopub.status.idle":"2022-05-05T13:41:50.094104Z","shell.execute_reply.started":"2022-05-05T13:41:50.052918Z","shell.execute_reply":"2022-05-05T13:41:50.093233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_images_for_feature_extraction(classifier, classification_type, img_process_function, df_train, df_test, img_dir, img_size, batch_size, validation_split = 0.15):\n\n    from tensorflow.keras.preprocessing.image import ImageDataGenerator\n    \n    from skimage import exposure\n    \n    # Defined image preprocessing functions\n\n    def preprocess_function(img):\n        \n        if img_process_function == \"equalize_adapthist\":\n            img = exposure.equalize_adapthist(img/255, clip_limit=0.03, kernel_size=24)\n        elif img_process_function == \"equalize_hist\":\n            img = exposure.equalize_hist(img/255, clip_limit=0.03, kernel_size=24)\n        elif img_process_function == \"rescale_intensity\":\n            img = exposure.rescale_intensity(img/255, clip_limit=0.03, kernel_size=24)\n            \n        return img\n    \n    \n    if classification_type == \"binary\":\n        y_col = \"image_label\"\n    else:\n        y_col = \"study_label\"\n    \n    \n    image_generator_train = ImageDataGenerator(preprocessing_function=preprocess_function,\n                                               validation_split=validation_split)\n        \n    image_generator_test = ImageDataGenerator(validation_split=validation_split,\n                                               preprocessing_function=preprocess_function)\n      \n\n    train_generator = image_generator_train.flow_from_dataframe(\n            dataframe = df_train,\n            directory=img_dir,\n            x_col = 'id',\n            y_col =  y_col,  \n            target_size=(img_size, img_size),\n            batch_size=batch_size,\n            seed = 42, \n            class_mode = \"raw\") \n    \n    test_generator = image_generator_test.flow_from_dataframe(\n            dataframe = df_test,\n            directory=img_dir,\n            x_col = 'id',\n            y_col = y_col,\n            target_size=(img_size, img_size),\n            batch_size=batch_size,\n            shuffle = False,  \n            seed=42, \n            class_mode = \"raw\")\n    \n    return train_generator, test_generator\n\ntrain_generator, test_generator = generate_images_for_feature_extraction( classifier = classifier, \n                                                                           classification_type = classification_type, \n                                                                           img_process_function = img_process_function, \n                                                                           df_train = df_train, \n                                                                           df_test = df_test, \n                                                                           img_dir = img_dir, \n                                                                           img_size = img_size, \n                                                                           batch_size = batch_size, \n                                                                           validation_split = 0.15)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:41:50.095515Z","iopub.execute_input":"2022-05-05T13:41:50.095873Z","iopub.status.idle":"2022-05-05T13:41:51.919956Z","shell.execute_reply.started":"2022-05-05T13:41:50.095839Z","shell.execute_reply":"2022-05-05T13:41:51.919104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Prepare Images for SVM Classifier","metadata":{}},{"cell_type":"code","source":"def prepare_images_for_SVM(train_generator, test_generator, train_num, val_num):\n\n\n    x_list = []\n    y_list = []\n    \n    for i in range(train_num):\n        x, y = next(train_generator)\n        x_list.append(x)\n        y_list.append(y)\n            \n    args = (x_list[i] for i in range(train_num))\n    x_tr = np.vstack((args))\n    args = (y_list[i] for i in range(train_num))\n    y_tr = np.vstack(args)\n    y_tr = y_tr.ravel()\n            \n    x_list = []\n    y_list = []\n    \n    for i in range(val_num):\n        x, y = next(test_generator)\n        x_list.append(x)\n        y_list.append(y)\n                \n    args = (x_list[i] for i in range(val_num))\n    x_val = np.vstack((args))\n    args = (y_list[i] for i in range(val_num))\n    y_val = np.vstack(args)\n    y_val = y_val.ravel()\n            \n    return x_tr, x_val, y_tr, y_val\n\nx_tr, x_val, y_tr, y_val = prepare_images_for_SVM(train_generator, test_generator, train_num, val_num)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:41:51.921607Z","iopub.execute_input":"2022-05-05T13:41:51.921907Z","iopub.status.idle":"2022-05-05T13:42:14.689865Z","shell.execute_reply.started":"2022-05-05T13:41:51.921866Z","shell.execute_reply":"2022-05-05T13:42:14.689246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Extracts Features from Generated Images","metadata":{}},{"cell_type":"code","source":"def extract_features_from_images(model, x_tr, x_val, y_tr, y_val):\n        \n    x_train = model.predict(x_tr)\n    x_test = model.predict(x_val)\n    y_train = y_tr\n    y_test = y_val\n        \n    return x_train, x_test, y_train, y_test\n\n# Extract feature vectors\nx_train, x_test, y_train, y_test = extract_features_from_images(model, x_tr, x_val, y_tr, y_val)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:42:14.691242Z","iopub.execute_input":"2022-05-05T13:42:14.691959Z","iopub.status.idle":"2022-05-05T13:42:49.885509Z","shell.execute_reply.started":"2022-05-05T13:42:14.69192Z","shell.execute_reply":"2022-05-05T13:42:49.884816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Print feature vectors' shapes","metadata":{}},{"cell_type":"code","source":"def print_feature_shapes(x_train, x_test, y_train, y_test):\n    \n    print(\"Extracted data shapes from transfer network\")\n    print(\"x_train shape: \",x_train.shape)\n    print(\"x_test shape: \",x_test.shape)\n    print(\"y_train shape: \",y_train.shape)\n    print(\"y_test shape: \",y_test.shape)\n    \nprint_feature_shapes(x_train, x_test, y_train, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:42:49.887321Z","iopub.execute_input":"2022-05-05T13:42:49.88789Z","iopub.status.idle":"2022-05-05T13:42:49.896278Z","shell.execute_reply.started":"2022-05-05T13:42:49.887847Z","shell.execute_reply":"2022-05-05T13:42:49.895464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Fit SVM cv models","metadata":{}},{"cell_type":"code","source":"def fit_cross_models(x_train, x_test, y_train, y_test, svm_hyp_search):\n    \n    if svm_hyp_search == \"grid\":\n        \n        svc_param_grid = {\"kernel\" : [\"rbf\", \"poly\", \"linear\"],\n                          \"gamma\": [1, 10],\n                          \"C\": [0.1, 10],\n                          \"degree\": [0.1, 10]}\n        \n        clf = GridSearchCV(SVC(random_state = 42), param_grid = svc_param_grid, \n                           cv = StratifiedKFold(n_splits = 5), scoring = \"accuracy\", \n                           n_jobs = -1,verbose = 1)\n        \n        \n    else:\n        \n        from skopt import BayesSearchCV\n        \n        # log-uniform: understand as search over p = exp(x) by varying x\n        \n        search_spaces = {\n                        'C': (1e-3, 1e+3, 'log-uniform'),\n                        'gamma': (1e-3, 1e+3, 'log-uniform'),\n                        'degree': (1, 5),  # integer valued parameter\n                        'kernel': ['linear', 'poly', 'rbf'],  # categorical parameter\n                        }\n        \n        clf = BayesSearchCV(\n            SVC(random_state = 42),\n            search_spaces=search_spaces,\n            n_iter=32,\n            cv=3)\n\n    clf.fit(x_train, y_train)\n        \n    svc = clf.best_estimator_\n    svc.fit(x_train, y_train)\n    y_pred = svc.predict(x_test)\n    \n    return clf, svc, y_pred\n\nclf, svc, y_pred = fit_cross_models(x_train, x_test, y_train, y_test, svm_hyp_search)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:42:49.898034Z","iopub.execute_input":"2022-05-05T13:42:49.89845Z","iopub.status.idle":"2022-05-05T13:43:47.661858Z","shell.execute_reply.started":"2022-05-05T13:42:49.898293Z","shell.execute_reply":"2022-05-05T13:43:47.660826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prints best SVM scores and estimator\n    \ndef print_best_results(clf, svc, x_train, y_train, x_test, y_test):\n    \n    train_accuracy = svc.score(x_train, y_train)\n    test_accuracy = svc.score(x_test, y_test)\n    \n    print(\"Best SVM estimator (parameters): \", clf.best_estimator_)\n    print(\"SVM Best Train accuracy\",train_accuracy)\n    print(\"SVM Best Test Accuracy\",test_accuracy)\n    \nprint_best_results(clf, svc, x_train, y_train, x_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:43:47.663448Z","iopub.execute_input":"2022-05-05T13:43:47.663765Z","iopub.status.idle":"2022-05-05T13:43:47.960317Z","shell.execute_reply.started":"2022-05-05T13:43:47.663719Z","shell.execute_reply":"2022-05-05T13:43:47.95919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Plots cross validation (optional)","metadata":{}},{"cell_type":"code","source":"if svm_hyp_search == \"grid\":\n    def plot_cv_splits(clf, number_of_top = 7):\n\n        results_df = pd.DataFrame(clf.cv_results_)\n        results_df = results_df.sort_values(by=['rank_test_score'])\n        results_df = (\n            results_df\n            .set_index(results_df[\"params\"].apply(\n                lambda x: \"_\".join(str(val) for val in x.values()))\n            )\n            .rename_axis('kernel')\n        )\n\n        # create df of model scores ordered by performance\n\n        model_scores = results_df.filter(regex=r'split\\d*_test_score')\n        model_scores = model_scores.transpose().iloc[:30,:number_of_top]\n\n        # plot 30 examples of dependency between cv fold and AUC scores\n\n        plt.subplots(figsize = (8,12))\n        sns.lineplot(\n            data=model_scores,\n            dashes=False, palette='Set1', marker='o', alpha=.5\n        )\n        plt.xlabel(\"CV test fold\")\n        plt.ylabel(\"Model AUC\")\n        plt.xticks(rotation = 45)\n        plt.show()\n\n    def plot_cv_scores(clf, max_rank = 10):\n\n        cv_results = pd.DataFrame(clf.cv_results_)\n        cv_results = cv_results[\n            ['params', 'rank_test_score', 'mean_test_score', 'std_test_score']\n        ]\n        cv_results = cv_results[cv_results['rank_test_score'] < max_rank]\n\n        cv_results = (\n            cv_results\n            .set_index(cv_results[\"params\"].apply(\n                lambda x: \"_\".join(str(val) for val in x.values()))\n            )\n            .rename_axis('kernel')\n        )\n\n        plt.figure(figsize = (10,8))\n        sns.lineplot(data=cv_results['mean_test_score'])\n        plt.xticks(rotation = 45)\n        plt.xlabel(\"Parameters\")\n        plt.ylabel(\"Mean test score\")\n        plt.show()\n\n    if show_cv_scores:\n        plot_cv_splits(clf, number_of_top = 7)\n        plot_cv_scores(clf, max_rank = 10)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:43:47.961521Z","iopub.execute_input":"2022-05-05T13:43:47.961745Z","iopub.status.idle":"2022-05-05T13:43:47.974279Z","shell.execute_reply.started":"2022-05-05T13:43:47.961719Z","shell.execute_reply":"2022-05-05T13:43:47.972607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Shows confusion matrix for SVM classifier","metadata":{}},{"cell_type":"code","source":"def plot_svm_confusion_matrix(svc, x_test, y_test):\n    \n    actual =  y_test\n    preds = svc.predict(x_test)\n    cfmx = confusion_matrix(actual, preds)\n    acc = accuracy_score(actual, preds)    \n    \n    plt.figure()\n    sns.heatmap(cfmx, annot=True, cmap='plasma',\n        xticklabels=list(np.unique(y_test)),\n            fmt='.0f', \n            yticklabels=list(np.unique(y_test))\n            )\n    plt.xlabel(\"Predictions\")\n    plt.ylabel(\"True Labels\")\n    plt.show()\n    \nplot_svm_confusion_matrix(svc, x_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:43:47.976992Z","iopub.execute_input":"2022-05-05T13:43:47.977288Z","iopub.status.idle":"2022-05-05T13:43:48.290712Z","shell.execute_reply.started":"2022-05-05T13:43:47.977256Z","shell.execute_reply":"2022-05-05T13:43:48.290162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"actual =  y_test\npreds = svc.predict(x_test)\nfrom sklearn import preprocessing\nle = preprocessing.LabelEncoder()\n\n\ncfmx = confusion_matrix(actual, preds)\nacc = accuracy_score(actual, preds)    \nfrom sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score, classification_report, average_precision_score, f1_score, precision_score, recall_score\n\nprint(\"Accuracy: \",acc)\n#print(\"Specificity:\",spe)\nprint(\"Weighted Precision:\",precision_score(actual, preds, average = \"weighted\"))\nprint(\"Weighted Sensitivity *Recall*:\",recall_score(actual, preds, average = \"weighted\"))\nprint(\"Weighted F1 score:\",f1_score(actual, preds, average = \"weighted\"))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:43:48.291704Z","iopub.execute_input":"2022-05-05T13:43:48.291987Z","iopub.status.idle":"2022-05-05T13:43:48.374173Z","shell.execute_reply.started":"2022-05-05T13:43:48.291961Z","shell.execute_reply":"2022-05-05T13:43:48.37338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Weighted ROC AUC Score\",roc_auc_score(le.fit_transform(actual), le.fit_transform(preds), average=\"weighted\"))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T13:43:48.375235Z","iopub.execute_input":"2022-05-05T13:43:48.375484Z","iopub.status.idle":"2022-05-05T13:43:48.384313Z","shell.execute_reply.started":"2022-05-05T13:43:48.375454Z","shell.execute_reply":"2022-05-05T13:43:48.383482Z"},"trusted":true},"execution_count":null,"outputs":[]}]}