{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from fastai.basics import *\nfrom fastai.callback.all import *\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\nfrom fastai import *\nfrom fastai.tabular import *\nfrom fastai.tabular.all import *\n\n\nimport pydicom\n\nimport pandas as pd\n\n\n# use gpu by default if available\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-19T20:59:37.475018Z","iopub.execute_input":"2023-01-19T20:59:37.475426Z","iopub.status.idle":"2023-01-19T20:59:40.551494Z","shell.execute_reply.started":"2023-01-19T20:59:37.475388Z","shell.execute_reply":"2023-01-19T20:59:40.550284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/siim-isic-melanoma-classification/test.csv\")\n\nprint(len(train_df), len(test_df))","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.553543Z","iopub.execute_input":"2023-01-19T20:59:40.554114Z","iopub.status.idle":"2023-01-19T20:59:40.683057Z","shell.execute_reply.started":"2023-01-19T20:59:40.554081Z","shell.execute_reply":"2023-01-19T20:59:40.681779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['benign_malignant'] = train_df['benign_malignant'].apply(lambda x : 0 if (x == \"benign\") else 1)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.684533Z","iopub.execute_input":"2023-01-19T20:59:40.684861Z","iopub.status.idle":"2023-01-19T20:59:40.726044Z","shell.execute_reply.started":"2023-01-19T20:59:40.684831Z","shell.execute_reply":"2023-01-19T20:59:40.725033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop(['patient_id'],axis=1)\ndummy1 = pd.get_dummies(train_df['sex'], drop_first=True)\ntrain_df = pd.concat([train_df, dummy1], axis=1).drop('sex', axis=1)\ntrain_df.head()\ndummy2 = pd.get_dummies(train_df['anatom_site_general_challenge'])\ntrain_df = pd.concat([train_df, dummy2], axis=1).drop('anatom_site_general_challenge', axis=1)\ntrain_df.head()\ntrain_df = train_df.drop(\"diagnosis\", axis=1)\ntrain_df = train_df.drop(\"target\", axis=1)\ntrain_df[\"age_approx\"] = train_df[\"age_approx\"].fillna(train_df[\"age_approx\"].mean()) # fillna null value in Math column with mean of the Math","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.729049Z","iopub.execute_input":"2023-01-19T20:59:40.729581Z","iopub.status.idle":"2023-01-19T20:59:40.771795Z","shell.execute_reply.started":"2023-01-19T20:59:40.729539Z","shell.execute_reply":"2023-01-19T20:59:40.770759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.773332Z","iopub.execute_input":"2023-01-19T20:59:40.773683Z","iopub.status.idle":"2023-01-19T20:59:40.791308Z","shell.execute_reply.started":"2023-01-19T20:59:40.773652Z","shell.execute_reply":"2023-01-19T20:59:40.790444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['benign_malignant'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.792729Z","iopub.execute_input":"2023-01-19T20:59:40.793060Z","iopub.status.idle":"2023-01-19T20:59:40.803918Z","shell.execute_reply.started":"2023-01-19T20:59:40.793031Z","shell.execute_reply":"2023-01-19T20:59:40.802718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mal_ims = train_df[train_df['benign_malignant']==1]\nn_mal_ims = len(mal_ims)\nprint(\"Number of malignant images: {}\".format(n_mal_ims))\nmal_ims.head()\n\n#create a df of a subset of benevolent images\nben_ims_subset = train_df[train_df['benign_malignant']==0].sample(n=n_mal_ims, random_state=42)\nn_ben_ims = len(ben_ims_subset)\nprint(\"Number of benign images in subset: {}\".format(n_ben_ims))\n\n#concatenate the two together and check the result\ntrain_df = pd.concat([mal_ims, ben_ims_subset], axis=0)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.805400Z","iopub.execute_input":"2023-01-19T20:59:40.806662Z","iopub.status.idle":"2023-01-19T20:59:40.831839Z","shell.execute_reply.started":"2023-01-19T20:59:40.806618Z","shell.execute_reply":"2023-01-19T20:59:40.831093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# means = [train_df[col].mean() for col in train_df.drop(['image_name', 'benign_malignant'], axis=1)]\n# std_devs = [train_df[col].std() for col in train_df.drop(['image_name', 'benign_malignant'], axis=1)]\n\nmeans = train_df['age_approx'].mean()\nstd_devs = train_df['age_approx'].std()\nprint(means)\nprint(std_devs)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.833225Z","iopub.execute_input":"2023-01-19T20:59:40.833574Z","iopub.status.idle":"2023-01-19T20:59:40.840298Z","shell.execute_reply.started":"2023-01-19T20:59:40.833547Z","shell.execute_reply":"2023-01-19T20:59:40.839179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\ndef stat_scaler(tensor):\n     return (tensor - means) / std_devs\n\nstat_scaler(tf.constant([1, 2, 3, 4, 5, 6, 7, 8], dtype=tf.float32))","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.842038Z","iopub.execute_input":"2023-01-19T20:59:40.843195Z","iopub.status.idle":"2023-01-19T20:59:40.886915Z","shell.execute_reply.started":"2023-01-19T20:59:40.843154Z","shell.execute_reply":"2023-01-19T20:59:40.886141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_df.sample(frac = 0.9)\n\nrest = train_df.drop(train.index)\n\nval = rest.sample(frac = 0.5)\n\ntest = rest.drop(val.index)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.890321Z","iopub.execute_input":"2023-01-19T20:59:40.891296Z","iopub.status.idle":"2023-01-19T20:59:40.900399Z","shell.execute_reply.started":"2023-01-19T20:59:40.891253Z","shell.execute_reply":"2023-01-19T20:59:40.899470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train[:1000]\nval_df = val[:50]\ntest_df = test[:50]\n\nprint(\"train:\", len(train_df))\nprint(\"val:\", len(val_df))\nprint(\"test:\", len(test_df))","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.901809Z","iopub.execute_input":"2023-01-19T20:59:40.902222Z","iopub.status.idle":"2023-01-19T20:59:40.910615Z","shell.execute_reply.started":"2023-01-19T20:59:40.902193Z","shell.execute_reply":"2023-01-19T20:59:40.909672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom as dicom\nimport cv2   \nimport numpy as np\nfrom PIL import Image as im\nfrom numpy import asarray\n\npath_imgs = \"/kaggle/input/siim-isic-melanoma-classification/jpeg/train/\"\n\ndef read_dicom_image(img_name):\n    img_path = os.path.join(path_imgs + img_name + \".jpg\")\n    image = cv2.imread(img_path)\n    image = im.fromarray(image)\n    image = image.resize((224,224))\n    image = asarray(image)\n    return image","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.911835Z","iopub.execute_input":"2023-01-19T20:59:40.913006Z","iopub.status.idle":"2023-01-19T20:59:40.921395Z","shell.execute_reply.started":"2023-01-19T20:59:40.912970Z","shell.execute_reply":"2023-01-19T20:59:40.920530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_dicom_image(\"ISIC_2637011\").shape","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:40.922654Z","iopub.execute_input":"2023-01-19T20:59:40.923026Z","iopub.status.idle":"2023-01-19T20:59:41.723633Z","shell.execute_reply.started":"2023-01-19T20:59:40.922996Z","shell.execute_reply":"2023-01-19T20:59:41.722312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Install sitk library to read dicom files\n!pip install SimpleITK  \nimport pandas as pd\nimport os\nimport SimpleITK as sitk\nimport torch\nfrom torchvision import transforms, datasets\nfrom torch.utils.data import Dataset, DataLoader\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:41.725540Z","iopub.execute_input":"2023-01-19T20:59:41.726305Z","iopub.status.idle":"2023-01-19T20:59:54.079979Z","shell.execute_reply.started":"2023-01-19T20:59:41.726260Z","shell.execute_reply":"2023-01-19T20:59:54.078800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class getImages(Dataset):\n    def __init__(self, frame, id_col):\n\n        self.frame = frame\n        self.id_col = id_col\n\n    def __len__(self):\n        return (self.frame.shape[0])\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n        #complete image path and read\n        img_name = self.frame[self.id_col].iloc[idx]\n        image = read_dicom_image(img_name)\n\n        return image","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:54.081864Z","iopub.execute_input":"2023-01-19T20:59:54.082224Z","iopub.status.idle":"2023-01-19T20:59:54.089437Z","shell.execute_reply.started":"2023-01-19T20:59:54.082190Z","shell.execute_reply":"2023-01-19T20:59:54.088241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_imgs = getImages(train_df, 'image_name')\nval_imgs = getImages(val_df, 'image_name')\ntest_imgs = getImages(test_df, 'image_name')","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:54.091280Z","iopub.execute_input":"2023-01-19T20:59:54.091621Z","iopub.status.idle":"2023-01-19T20:59:54.103732Z","shell.execute_reply.started":"2023-01-19T20:59:54.091591Z","shell.execute_reply":"2023-01-19T20:59:54.102782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image1 = train_imgs[0]\ntrain_images = []\nif test is None:\n    print(f'This image is bad')\nelse:\n    train_images.append(image1)\ntrain_images = np.array(train_images)\nfor i in range(1, len(train_imgs)):\n    train_images = np.vstack([train_images, [train_imgs[i]]])\n    \nimage1 = val_imgs[0]\nval_images = []\nif test is None:\n    print(f'This image is bad')\nelse:\n    val_images.append(image1)\nval_images = np.array(val_images)\nfor i in range(1, len(val_imgs)):\n    val_images = np.vstack([val_images, [val_imgs[i]]])\n    \nimage1 = test_imgs[0]\ntest_images = []\nif test is None:\n    print(f'This image is bad')\nelse:\n    test_images.append(image1)\ntest_images = np.array(test_images)\nfor i in range(1, len(test_imgs)):\n    test_images = np.vstack([test_images, [test_imgs[i]]])","metadata":{"execution":{"iopub.status.busy":"2023-01-19T20:59:54.105011Z","iopub.execute_input":"2023-01-19T20:59:54.105351Z","iopub.status.idle":"2023-01-19T21:06:22.217144Z","shell.execute_reply.started":"2023-01-19T20:59:54.105320Z","shell.execute_reply":"2023-01-19T21:06:22.215832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feats  = np.array(train_df[['age_approx','male', 'head/neck', 'lower extremity', 'oral/genital', 'palms/soles', 'torso', 'upper extremity']])\nval_feats  = np.array(val_df[['age_approx','male', 'head/neck', 'lower extremity', 'oral/genital', 'palms/soles', 'torso', 'upper extremity']])\ntest_feats  = np.array(test_df[['age_approx','male', 'head/neck', 'lower extremity', 'oral/genital', 'palms/soles', 'torso', 'upper extremity']])\n\ntrain_labels = np.array(train_df['benign_malignant'])\nval_labels = np.array(val_df['benign_malignant'])\ntest_labels = np.array(test_df['benign_malignant'])","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:06:22.218778Z","iopub.execute_input":"2023-01-19T21:06:22.219144Z","iopub.status.idle":"2023-01-19T21:06:22.233024Z","shell.execute_reply.started":"2023-01-19T21:06:22.219111Z","shell.execute_reply":"2023-01-19T21:06:22.231649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train_data\")\nprint(\"images shape:\", train_images.shape)\nprint(\"feats shape:\", train_feats.shape)\nprint(\"labels shape:\", train_labels.shape)\n\nprint(\"\")\nprint(\"val_data\")\nprint(\"images shape:\", val_images.shape)\nprint(\"feats shape:\", val_feats.shape)\nprint(\"labels shape:\", val_labels.shape)\n\nprint(\"\")\nprint(\"test_data\")\nprint(\"images shape:\", test_images.shape)\nprint(\"feats shape:\", test_feats.shape)\nprint(\"labels shape:\", test_labels.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:06:22.235096Z","iopub.execute_input":"2023-01-19T21:06:22.236254Z","iopub.status.idle":"2023-01-19T21:06:22.247517Z","shell.execute_reply.started":"2023-01-19T21:06:22.236198Z","shell.execute_reply":"2023-01-19T21:06:22.246143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"font-size:32px; font-family:Verdana\"> CNN Model with Just Pictures (Baseline) </span>","metadata":{}},{"cell_type":"code","source":"# Define the Model\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Model\n\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.applications.mobilenet_v2 import preprocess_input\nfrom tensorflow.keras import regularizers\n\ninput_pic = layers.Input(shape=(224, 224, 3))\nx         = layers.Lambda(preprocess_input)(input_pic)\nx         = MobileNetV2(input_shape=((224, 224, 3)), include_top=False, weights='imagenet')(x)\nx.trainable = False\nx         = layers.GlobalAveragePooling2D()(x)\nx         = layers.Flatten()(x)\nx         = layers.Dense(10, kernel_regularizer=regularizers.L2(0.1),activity_regularizer=regularizers.L1(0.1), activation='relu')(x)\nx         = layers.Dense(1, activation='sigmoid')(x)\nmodel     = Model(inputs=input_pic, outputs=x)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:06:22.249243Z","iopub.execute_input":"2023-01-19T21:06:22.249816Z","iopub.status.idle":"2023-01-19T21:06:24.544718Z","shell.execute_reply.started":"2023-01-19T21:06:22.249772Z","shell.execute_reply":"2023-01-19T21:06:24.543539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:06:24.546586Z","iopub.execute_input":"2023-01-19T21:06:24.546940Z","iopub.status.idle":"2023-01-19T21:06:24.563520Z","shell.execute_reply.started":"2023-01-19T21:06:24.546908Z","shell.execute_reply":"2023-01-19T21:06:24.562313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam\n\nlearning_rate = .0001\noptimizer = Adam(learning_rate=learning_rate)\n\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer, metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:06:24.565153Z","iopub.execute_input":"2023-01-19T21:06:24.565886Z","iopub.status.idle":"2023-01-19T21:06:24.585665Z","shell.execute_reply.started":"2023-01-19T21:06:24.565840Z","shell.execute_reply":"2023-01-19T21:06:24.584792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(x= train_images, y = train_labels,\n          validation_data = (val_images, val_labels),\n          batch_size= 100,\n          epochs=8,\n          verbose=2)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:06:24.587265Z","iopub.execute_input":"2023-01-19T21:06:24.587975Z","iopub.status.idle":"2023-01-19T21:15:12.455102Z","shell.execute_reply.started":"2023-01-19T21:06:24.587930Z","shell.execute_reply":"2023-01-19T21:15:12.453984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:17:31.210225Z","iopub.execute_input":"2023-01-19T21:17:31.210645Z","iopub.status.idle":"2023-01-19T21:17:31.654218Z","shell.execute_reply.started":"2023-01-19T21:17:31.210608Z","shell.execute_reply":"2023-01-19T21:17:31.652616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"font-size:32px; font-family:Verdana\"> Multi-input Model </span>","metadata":{}},{"cell_type":"code","source":"# Define the Model\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Model\n\nfrom tensorflow.keras.applications import MobileNetV2\nfrom tensorflow.keras.applications.mobilenet_v2 import preprocess_input\nfrom tensorflow.keras import regularizers\n\n# Define the Picture (CNN) Stream\n\ninput_pic = layers.Input(shape=(224, 224, 3))\nx         = layers.Lambda(preprocess_input)(input_pic)\nx         = MobileNetV2(input_shape=((224, 224, 3)), include_top=False, weights='imagenet')(x)\nx.        trainable = False\nx         = layers.GlobalAveragePooling2D()(x)\nx         = layers.Flatten()(x)\nx         = layers.Dense(10, kernel_regularizer=regularizers.L2(0.1),activity_regularizer=regularizers.L1(0.1), activation='relu')(x)\nx         = Model(inputs=input_pic, outputs=x)\n\n# Define the Stats (Feed-Forward) Stream\n\ninput_stats = layers.Input(shape=(8,))\n# y = layers.Lambda(stat_scaler)(input_stats)\ny = input_stats\ny = layers.Dense(32, kernel_regularizer=regularizers.L2(0.1),\n    activity_regularizer=regularizers.L1(.1), activation=\"relu\")(y)\n# y = layers.Dense(32, activation=\"relu\")(y)\ny = layers.Dense(2, activation=\"relu\")(y)\ny = Model(inputs=input_stats, outputs=y)\n\n# Concatenate the two streams together\ncombined = layers.concatenate([x.output, y.output])\n\n# Define joined Feed-Forward Layer\nz = layers.Dense(5, activation=\"relu\")(combined)\n\n# Define output node of 1 linear neuron (regression task)\nz = layers.Dense(1, activation=\"sigmoid\")(z)\n\n# Define the final model\nmodel = Model(inputs=[x.input, y.input], outputs=z)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T22:24:43.632051Z","iopub.execute_input":"2023-01-19T22:24:43.632469Z","iopub.status.idle":"2023-01-19T22:24:45.226844Z","shell.execute_reply.started":"2023-01-19T22:24:43.632433Z","shell.execute_reply":"2023-01-19T22:24:45.225725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam\n\nlearning_rate = .0001\noptimizer = Adam(learning_rate=learning_rate)\n\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer, metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-01-19T22:33:22.923198Z","iopub.execute_input":"2023-01-19T22:33:22.923746Z","iopub.status.idle":"2023-01-19T22:33:22.965340Z","shell.execute_reply.started":"2023-01-19T22:33:22.923689Z","shell.execute_reply":"2023-01-19T22:33:22.963913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint\n\ncp = ModelCheckpoint('model/', save_best_only=True)\n\nhistory = model.fit(x=[train_images, train_feats], y = train_labels,\n          validation_data=([val_images, val_feats], val_labels),\n          batch_size= 100,\n          epochs=10,\n          verbose=2,\n          callbacks=[cp])        ","metadata":{"execution":{"iopub.status.busy":"2023-01-19T22:33:25.189042Z","iopub.execute_input":"2023-01-19T22:33:25.190006Z","iopub.status.idle":"2023-01-19T22:47:28.529261Z","shell.execute_reply.started":"2023-01-19T22:33:25.189935Z","shell.execute_reply":"2023-01-19T22:47:28.528051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for accuracy\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:55:48.429753Z","iopub.execute_input":"2023-01-19T21:55:48.430220Z","iopub.status.idle":"2023-01-19T21:55:48.856224Z","shell.execute_reply.started":"2023-01-19T21:55:48.430176Z","shell.execute_reply":"2023-01-19T21:55:48.854924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions = model.predict([test_images, test_feats])\ntest_predictions[:10]","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:33:46.873765Z","iopub.execute_input":"2023-01-19T21:33:46.875094Z","iopub.status.idle":"2023-01-19T21:33:49.374589Z","shell.execute_reply.started":"2023-01-19T21:33:46.875027Z","shell.execute_reply":"2023-01-19T21:33:49.373203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(test_predictions)):\n    if (test_predictions[i][0]>0.5):\n        test_predictions[i] = 1\n    else:\n        test_predictions[i] = 0\ntest_predictions","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:33:56.578577Z","iopub.execute_input":"2023-01-19T21:33:56.579102Z","iopub.status.idle":"2023-01-19T21:33:56.589935Z","shell.execute_reply.started":"2023-01-19T21:33:56.579056Z","shell.execute_reply":"2023-01-19T21:33:56.589056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ncm = confusion_matrix(y_true=test_labels, y_pred=test_predictions)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:34:00.535105Z","iopub.execute_input":"2023-01-19T21:34:00.535777Z","iopub.status.idle":"2023-01-19T21:34:00.543573Z","shell.execute_reply.started":"2023-01-19T21:34:00.535712Z","shell.execute_reply":"2023-01-19T21:34:00.542427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                        normalize=False,\n                        title='Confusion matrix',\n                        cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        print(\"Normalized confusion matrix\")\n    else:\n        print('Confusion matrix, without normalization')\n\n    print(cm)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n            horizontalalignment=\"center\",\n            color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:34:02.045629Z","iopub.execute_input":"2023-01-19T21:34:02.046077Z","iopub.status.idle":"2023-01-19T21:34:02.056202Z","shell.execute_reply.started":"2023-01-19T21:34:02.046039Z","shell.execute_reply":"2023-01-19T21:34:02.055237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm_plot_labels = ['benign','malignant']\nplot_confusion_matrix(cm=cm, classes=cm_plot_labels, title='Confusion Matrix')","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:34:04.583235Z","iopub.execute_input":"2023-01-19T21:34:04.583657Z","iopub.status.idle":"2023-01-19T21:34:04.865639Z","shell.execute_reply.started":"2023-01-19T21:34:04.583622Z","shell.execute_reply":"2023-01-19T21:34:04.864602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm = confusion_matrix(y_true=test_labels, y_pred=test_predictions)\ncm_plot_labels = ['benign','malignant']\nplot_confusion_matrix(cm=cm, classes=cm_plot_labels, title='Confusion Matrix')\n\ntp = cm[0][0]\nfp = cm[0][1]\nfn = cm[1][0]\ntn = cm[1][1]\n\nsensitivity = tp/(tp + fn)\nspecificity = tn/(fp + tn)\nppv = tp/(tp+fp)\nnpv = tn/(tn+fn)\naccuracy = (tp + tn)/(tp + fp + fn + tn)\n\nprint(\"Sensitivity: \", sensitivity)\nprint(\"Specificity: \", specificity)\nprint(\"Positive Predictive Value: \", ppv)\nprint(\"Negative Predicitve Value: \", npv)\nprint(\"Accuracy: \", accuracy)","metadata":{"execution":{"iopub.status.busy":"2023-01-19T21:34:07.974328Z","iopub.execute_input":"2023-01-19T21:34:07.975129Z","iopub.status.idle":"2023-01-19T21:34:08.256389Z","shell.execute_reply.started":"2023-01-19T21:34:07.975085Z","shell.execute_reply":"2023-01-19T21:34:08.255266Z"},"trusted":true},"execution_count":null,"outputs":[]}]}