{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ! conda install -y gdown","metadata":{"id":"4AcaURGGG6Uv","execution":{"iopub.status.busy":"2021-05-21T15:34:00.158085Z","iopub.execute_input":"2021-05-21T15:34:00.158533Z","iopub.status.idle":"2021-05-21T15:34:00.165976Z","shell.execute_reply.started":"2021-05-21T15:34:00.158497Z","shell.execute_reply":"2021-05-21T15:34:00.164915Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !gdown https://drive.google.com/uc?id=1FKJULb6GRAM_GLvQzhxUhWIgGydt4Mtf","metadata":{"execution":{"iopub.status.busy":"2021-05-21T15:34:00.222447Z","iopub.execute_input":"2021-05-21T15:34:00.222898Z","iopub.status.idle":"2021-05-21T15:34:00.227390Z","shell.execute_reply.started":"2021-05-21T15:34:00.222839Z","shell.execute_reply":"2021-05-21T15:34:00.226117Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load dependencies\nfrom __future__ import print_function\nimport tensorflow as tf   # Using tensorflow 2.0.0\nfrom keras import layers, initializers\nfrom keras import backend as K\nfrom keras import activations\nfrom keras import utils\nfrom keras.models import Model\nfrom keras.layers import *\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications.vgg16 import VGG16, preprocess_input\nfrom keras.applications.vgg16 import VGG16, preprocess_input\nfrom keras.optimizers import RMSprop, Adam, SGD, Nadam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\nfrom keras import regularizers\nfrom keras.models import load_model\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\n\nIMG_SIZE = 299","metadata":{"execution":{"iopub.status.busy":"2021-05-21T15:34:00.357346Z","iopub.execute_input":"2021-05-21T15:34:00.357743Z","iopub.status.idle":"2021-05-21T15:34:00.368194Z","shell.execute_reply.started":"2021-05-21T15:34:00.357708Z","shell.execute_reply":"2021-05-21T15:34:00.366790Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def DataGenerator(train_batch, val_batch, IMG_SIZE):\n    datagen = ImageDataGenerator(\n                                 rescale=1./255,#rescale factor\n                                 rotation_range=10,# Degree range for random rotations.\n                                 horizontal_flip=True,#horizontal flip in images\n                                 vertical_flip=True)#vertical flip in images\n\n    datagen.mean=np.array([103.939, 116.779, 123.68],dtype=np.float32).reshape(1,1,3)\n\n    train_gen = datagen.flow_from_directory('../input/cross2/cross2/Train/',\n                                            target_size=(IMG_SIZE, IMG_SIZE),\n                                            color_mode='rgb', \n                                            class_mode='categorical',\n                                            batch_size=train_batch)\n\n    val_gen = datagen.flow_from_directory('../input/cross2/cross2/Val/', \n                                          target_size=(IMG_SIZE, IMG_SIZE),\n                                          color_mode='rgb', \n                                          class_mode='categorical',\n                                          batch_size=val_batch)\n\n    datagen = ImageDataGenerator(rescale=1./255)\n    \n    datagen.mean=np.array([103.939, 116.779, 123.68],dtype=np.float32).reshape(1,1,3)\n\n    test_gen = datagen.flow_from_directory('../input/cross2/cross2/Test/', \n                                           target_size=(IMG_SIZE, IMG_SIZE),\n                                           color_mode='rgb', \n                                           class_mode='categorical',\n                                           shuffle=False)\n    \n    return train_gen, val_gen, test_gen","metadata":{"id":"x05TXAgMHHx2","execution":{"iopub.status.busy":"2021-05-21T15:34:00.727924Z","iopub.execute_input":"2021-05-21T15:34:00.728347Z","iopub.status.idle":"2021-05-21T15:34:00.739636Z","shell.execute_reply.started":"2021-05-21T15:34:00.728309Z","shell.execute_reply":"2021-05-21T15:34:00.738308Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_weight = {0: 1,\n                1: 3,\n                2: 3,\n                3: 7,\n                4: 7}","metadata":{"execution":{"iopub.status.busy":"2021-05-21T15:34:00.747041Z","iopub.execute_input":"2021-05-21T15:34:00.747528Z","iopub.status.idle":"2021-05-21T15:34:00.758116Z","shell.execute_reply.started":"2021-05-21T15:34:00.747493Z","shell.execute_reply":"2021-05-21T15:34:00.756831Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the squashing function.\n\"\"\"\nThe non-linear activation used in Capsule. It drives the length of a large vector to near 1 and small vector to 0\n:param vectors: some vectors to be squashed, N-dim tensor\n:param axis: the axis to squash\n:return: a Tensor with same shape as input vectors\n\"\"\"\ndef squash(x, axis=-1):\n    s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n    scale = K.sqrt(s_squared_norm) / (0.5 + s_squared_norm)\n    return scale * x","metadata":{"id":"SzPeeXXMHI7x","execution":{"iopub.status.busy":"2021-05-21T15:34:00.892390Z","iopub.execute_input":"2021-05-21T15:34:00.892782Z","iopub.status.idle":"2021-05-21T15:34:00.901271Z","shell.execute_reply.started":"2021-05-21T15:34:00.892733Z","shell.execute_reply":"2021-05-21T15:34:00.899644Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define our own softmax function instead of K.softmax\n# because K.softmax can not specify axis.\ndef softmax(x, axis=-1):\n    ex = K.exp(x - K.max(x, axis=axis, keepdims=True))\n    return ex / K.sum(ex, axis=axis, keepdims=True)","metadata":{"id":"EyBPnMj0HI_u","execution":{"iopub.status.busy":"2021-05-21T15:34:00.967013Z","iopub.execute_input":"2021-05-21T15:34:00.967451Z","iopub.status.idle":"2021-05-21T15:34:00.974338Z","shell.execute_reply.started":"2021-05-21T15:34:00.967416Z","shell.execute_reply":"2021-05-21T15:34:00.972854Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def caps_batch_dot(x, y):\n    x = K.expand_dims(x, 2)\n    if K.int_shape(x)[3] is not None:\n        y = K.permute_dimensions(y, (0, 1, 3, 2))\n    o = tf.matmul(x, y)\n    return K.squeeze(o, 2)\n\nclass Capsule(Layer):\n    \"\"\"\n    The capsule layer. It is similar to Dense layer. Dense layer has `in_num` inputs, each is a scalar, the output of the \n    neuron from the former layer, and it has `out_num` output neurons. CapsuleLayer just expand the output of the neuron\n    from scalar to vector. So its input shape = [None, input_num_capsule, input_dim_capsule] and output shape = \\\n    [None, num_capsule, dim_capsule]. For Dense Layer, input_dim_capsule = dim_capsule = 1.\n    \n    :param num_capsule: number of capsules in this layer\n    :param dim_capsule: dimension of the output vectors of the capsules in this layer\n    :param routings: number of iterations for the routing algorithm\n    \"\"\"\n    def __init__(self,\n                 num_capsule,\n                 dim_capsule,\n                 routings=3,\n                 share_weights=True,\n                 activation='squash',\n                 **kwargs):\n        super(Capsule, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.share_weights = share_weights\n        if activation == 'squash':\n            self.activation = squash\n        else:\n            self.activation = activations.get(activation)\n\n    def build(self, input_shape):\n        input_dim_capsule = input_shape[-1]\n        if self.share_weights:\n            self.kernel = self.add_weight(\n                name='capsule_kernel',\n                shape=(1, input_dim_capsule,\n                       self.num_capsule * self.dim_capsule),\n                initializer='glorot_uniform',\n                trainable=True)\n        else:\n            if input_shape[-2] is None:\n                raise ValueError(\"Input Shape must be defied if weights not shared.\")\n            input_num_capsule = input_shape[-2]\n            self.kernel = self.add_weight(\n                name='capsule_kernel',\n                shape=(input_num_capsule, input_dim_capsule,\n                       self.num_capsule * self.dim_capsule),\n                initializer='glorot_uniform',\n                trainable=True)\n\n    def call(self, inputs):\n        \"\"\"Following the routing algorithm from Hinton's paper,\n        but replace b = b + <u,v> with b = <u,v>.\n        This change can improve the feature representation of Capsule.\n        However, you can replace\n            b = K.batch_dot(outputs, hat_inputs, [2, 3])\n        with\n            b += K.batch_dot(outputs, hat_inputs, [2, 3])\n        to realize a standard routing.\n        \"\"\"\n\n        if self.share_weights:\n            hat_inputs = K.conv1d(inputs, self.kernel)\n        else:\n            hat_inputs = K.local_conv1d(inputs, self.kernel, [1], [1])\n\n        batch_size = K.shape(inputs)[0]\n        input_num_capsule = K.shape(inputs)[1]\n        hat_inputs = K.reshape(hat_inputs,\n                               (batch_size, input_num_capsule,\n                                self.num_capsule, self.dim_capsule))\n        hat_inputs = K.permute_dimensions(hat_inputs, (0, 2, 1, 3))\n\n        b = K.zeros_like(hat_inputs[:, :, :, 0])\n\n        \n        # Begin: Routing algorithm ---------------------------------------------------------------------#\n        # The prior for coupling coefficient, initialized as zeros.\n        # b.shape = [None, self.num_capsule, self.input_num_capsule].\n\n        for i in range(self.routings):\n            c = softmax(b, 1)\n            o = self.activation(caps_batch_dot(c, hat_inputs))\n            if i < self.routings - 1:\n                b = caps_batch_dot(o, hat_inputs)\n                if K.backend() == 'theano':\n                    o = K.sum(o, axis=1)\n        # End: Routing algorithm -----------------------------------------------------------------------#\n        return o\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.num_capsule, self.dim_capsule)\n        \n    def get_config(self):\n        config = {\n            'num_capsule': self.num_capsule,\n            'dim_capsule': self.dim_capsule,\n            'routings': self.routings\n        }\n        base_config = super(Capsule, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","metadata":{"id":"saY31CkNHJC3","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-05-21T15:34:01.267134Z","iopub.execute_input":"2021-05-21T15:34:01.267677Z","iopub.status.idle":"2021-05-21T15:34:01.289021Z","shell.execute_reply.started":"2021-05-21T15:34:01.267642Z","shell.execute_reply":"2021-05-21T15:34:01.287261Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Generating data for training using Image Generator defined above\ntrain_batch = 32\nval_batch = 1\n\ntrain, val, test = DataGenerator(train_batch, val_batch, IMG_SIZE)","metadata":{"id":"o9-b_xyIHJFs","outputId":"91791a77-3c4f-4a25-8276-28d9b13de328","execution":{"iopub.status.busy":"2021-05-21T15:34:01.407284Z","iopub.execute_input":"2021-05-21T15:34:01.407676Z","iopub.status.idle":"2021-05-21T15:34:01.833408Z","shell.execute_reply.started":"2021-05-21T15:34:01.407643Z","shell.execute_reply":"2021-05-21T15:34:01.832340Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers import Activation, Dropout, Flatten, Dense\ninput_image = Input(shape=(IMG_SIZE, IMG_SIZE, 3))\n\n\n# A InceptionResNetV2 Conv2D model\n# model = VGG16(include_top=False, weights='imagenet', input_tensor=input_image)\n# base_model = tf.keras.applications.ResNet152(include_top=False, weights='imagenet',input_tensor=input_image)\n# base_model =  tf.compat.v2.keras.applications.Xception(include_top=False, weights='imagenet', input_tensor=input_image)\n# base_model = tf.keras.applications.EfficientNetB1(include_top=False, weights='imagenet',input_tensor= input_image)                                                   \n\n\n#     base_model = tf.compat.v2.keras.applications.ResNet152(include_top=False, weights='imagenet', input_tensor=None, input_shape=IMG_SHAPE, pooling=None, classes=2)\n\n#     base_model = tf.compat.v2.keras.applications.ResNet152(include_top=False, weights='imagenet', input_tensor=None, input_shape=IMG_SHAPE, pooling=None, classes=2)\n\n#     base_model = tf.compat.v2.keras.applications.VGG16(include_top=False, weights='imagenet', input_tensor=None, input_shape=IMG_SHAPE, pooling=None, classes=3)\n#     base_model = tf.compat.v2.keras.applications.inception_v3.InceptionV3(include_top=False, weights='imagenet', input_tensor=None, input_shape=IMG_SHAPE, pooling=None, classes=3)\nbase_model =  tf.compat.v2.keras.applications.Xception(include_top=False, weights='imagenet', input_tensor=input_image)\n#     base_model = tf.compat.v2.keras.applications.VGG19(include_top=False, weights='imagenet', input_tensor=None, input_shape=IMG_SHAPE, pooling=None, classes=1000)\n# base_model = tf.keras.applications.resnet50.ResNet50(include_top=False, weights='imagenet', input_tensor=input_image)\n#     base_model = tf.keras.applications.EfficientNetB0(include_top=False, weights='imagenet',input_shape=IMG_SHAPE)\n#     base_model = tf.keras.applications.MobileNetV2(include_top=False, weights='imagenet',input_shape=IMG_SHAPE)\n                                            \n#     base_model = tf.keras.applications.InceptionResNetV2(include_top=False, weights='imagenet',input_tensor=None, input_shape=None,pooling=None, classes=1000)\n\n\n\n\n# base_model.trainable = True\n#     # Let's take a look to see how many layers are in the base model\n# print(\"Number of layers in the base model: \", len(base_model.layers))\n#     # Fine-tune from this layer onwards\n# fine_tune_at =1\n\n#     # Freeze all the layers before the `fine_tune_at` layer\n# for layer in base_model.layers[:fine_tune_at]:\n#       layer.trainable =  False\n# # base_model.summary()","metadata":{"id":"X8k9_5GvHJIY","outputId":"b72ddb4c-8ad7-48d0-ccd9-733c0ac99a1b","execution":{"iopub.status.busy":"2021-05-21T15:34:01.835777Z","iopub.execute_input":"2021-05-21T15:34:01.836255Z","iopub.status.idle":"2021-05-21T15:34:03.386206Z","shell.execute_reply.started":"2021-05-21T15:34:01.836203Z","shell.execute_reply":"2021-05-21T15:34:03.384294Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = Conv2D(256, kernel_size=(9, 9), strides=(1, 1), activation='relu')(base_model.get_layer(name='block14_sepconv2_act').output)\nx = Reshape((-1, 256))(output)\nx = Capsule(5, 16, 4, True)(x)\n# x= layers.Flatten()(x)\n# x = Dropout(0.12)(x)\n# x = layers.Dense(512, activation='relu')(x)\n# x = layers.Dense(1024, activation='relu')(x)\n# x = Dense(5, activation=\"softmax\")(x)\noutput = Lambda(lambda z: K.sqrt(K.sum(K.square(z), 2)))(x)\nmodel = Model(inputs=input_image,outputs=output)\n\n# model.summary()","metadata":{"id":"LD9t0ffcHJMW","outputId":"e79ae0e2-4329-421b-b626-d2a878cea7ba","execution":{"iopub.status.busy":"2021-05-21T15:34:03.389147Z","iopub.execute_input":"2021-05-21T15:34:03.389625Z","iopub.status.idle":"2021-05-21T15:34:03.595983Z","shell.execute_reply.started":"2021-05-21T15:34:03.389582Z","shell.execute_reply":"2021-05-21T15:34:03.594838Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr=1e-4\ncheckpoint = ModelCheckpoint(\"EfN_model-1.h5\", \n                             monitor='val_loss', \n                             verbose=1, \n                             save_best_only=True, \n                             save_weights_only=False, \n                             mode='min')\nearly = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min', restore_best_weights=True)\nrlr = ReduceLROnPlateau(monitor='val_loss', \n                        factor=0.85, \n                        patience=2, \n                        verbose=1, \n                        mode='auto', \n                        epsilon=0.0001)\ncallback_list = [checkpoint,rlr,early]","metadata":{"id":"VoBNeNDHHJPD","execution":{"iopub.status.busy":"2021-05-21T15:34:03.598477Z","iopub.execute_input":"2021-05-21T15:34:03.599091Z","iopub.status.idle":"2021-05-21T15:34:03.608940Z","shell.execute_reply.started":"2021-05-21T15:34:03.599044Z","shell.execute_reply":"2021-05-21T15:34:03.607275Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs=100\n\n# model.compile(loss=margin_loss, optimizer=SGD(lr=lr, momentum=0.9), metrics=['accuracy'])\n\n# model.compile(loss='categorical_crossentropy', \n#               optimizer=tf.keras.optimizers.RMSprop(learning_rate=0.00001, rho=0.9,momentum=0.0,epsilon=1e-07,\n#                                                     centered=False,name=\"RMSprop\"),  \n#               metrics=['accuracy'])\nmodel.compile(loss='categorical_crossentropy', \n              optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),  \n              metrics=['accuracy'])\n\n\n\nhistory=model.fit_generator(train,\n          epochs=epochs,\n          validation_data=val, \n          validation_steps = len(val.classes)//val_batch,              \n          steps_per_epoch=len(train.classes)//train_batch,\n           class_weight=class_weight,\n           callbacks=callback_list) \nmodel.save('EfN_model-1.h5')\n# evaluate the model\n_, train_acc = model.evaluate(train, verbose=1)\n_, test_acc = model.evaluate(test, verbose=1)\nprint('Train: %.3f, Test: %.3f' % (train_acc, test_acc))\nmodel1 = load_model('EfN_model-1.h5', custom_objects={'Capsule': Capsule})\n_, test_acc = model1.evaluate(test, verbose=1)\nprint('Train by EfN_model-1: %.3f' % (test_acc))\n\n# loss, acc = model.evaluate_generator(test, len(test))\n# print (\"\\n\\n================================\\n\\n\")\n# print (\"Loss: {}\".format(loss))\n# print (\"Accuracy: {0:.2f} %\".format(acc * 100))\n# print (\"\\n\\n================================\\n\\n\")\n# test.reset()","metadata":{"id":"t5QUvHZYHJSX","outputId":"94076eed-c3e4-48ea-bdb7-bbbd4936fb52","execution":{"iopub.status.busy":"2021-05-21T15:34:03.618438Z","iopub.execute_input":"2021-05-21T15:34:03.619141Z","iopub.status.idle":"2021-05-21T16:07:31.157607Z","shell.execute_reply.started":"2021-05-21T15:34:03.619093Z","shell.execute_reply":"2021-05-21T16:07:31.155381Z"},"editable":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import load_model\nimport tqdm\nmc_predictions = [] \nfor i in tqdm.tqdm(range(10)):\n    test.reset()\n    y_p = model.predict_generator(test, steps=None, callbacks=None, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=0)\n    #y_p = model.predict(test_2Images, batch_size=10)\n    mc_predictions.append(y_p)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-05-21T16:07:31.159099Z","iopub.execute_input":"2021-05-21T16:07:31.160749Z","iopub.status.idle":"2021-05-21T16:08:17.845751Z","shell.execute_reply.started":"2021-05-21T16:07:31.160697Z","shell.execute_reply":"2021-05-21T16:08:17.844055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom glob import glob\n#import matplotlib.pyplot as plt\n# stacked generalization with linear meta model on blobs dataset\nfrom sklearn.datasets import make_blobs\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.linear_model import LogisticRegression\nfrom keras.models import load_model\nfrom keras.utils import to_categorical\nfrom numpy import dstack\n#y_test=test_labels\ntest.reset()\ntrue_label = test.classes\n\n#y_test.argmax(axis=1)\n\n# score of the mc model\naccs = []\nfor y_p in mc_predictions:\n    acc = accuracy_score(true_label, y_p.argmax(axis=1))\n    accs.append(acc)\nprint(\"MC accuracy: {:.1%}\".format(sum(accs)/len(accs)))\n\n# ensemble \nmc_ensemble_pred = np.array(mc_predictions).mean(axis=0).argmax(axis=1)\nensemble_acc = accuracy_score(true_label, mc_ensemble_pred)\nprint(\"MC-ensemble accuracy: {:.1%}\".format(ensemble_acc))","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-05-21T16:08:17.847358Z","iopub.execute_input":"2021-05-21T16:08:17.847751Z","iopub.status.idle":"2021-05-21T16:08:17.870327Z","shell.execute_reply.started":"2021-05-21T16:08:17.847708Z","shell.execute_reply":"2021-05-21T16:08:17.868871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\ntype(mc_predictions)\n\nwith open(\"predictions.txt\", \"wb\") as fp:   #Pickling\n    pickle.dump(mc_predictions, fp)\n\nwith open(\"acc.txt\", \"wb\") as fp:   #Pickling\n    pickle.dump(accs, fp)\n    \nwith open(\"y_p.txt\", \"wb\") as fp:   #Pickling\n    pickle.dump(y_p, fp)   ","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-05-21T16:08:17.872570Z","iopub.execute_input":"2021-05-21T16:08:17.873376Z","iopub.status.idle":"2021-05-21T16:08:17.890127Z","shell.execute_reply.started":"2021-05-21T16:08:17.873326Z","shell.execute_reply":"2021-05-21T16:08:17.888478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import scipy.stats as stats\nimport math\n\n#np.random.seed(10)\n#population_ages = accs\nsample_size = len(accs)\npopulation_ages= sample = np.asarray(accs, dtype=np.float32)\nsample_mean = sample.mean()\n\nz_critical = stats.norm.ppf(q = 0.975)  # Get the z-critical value*\n\n#z_critical=1.96\nprint(\"z-critical value:\",z_critical)             \n                       \nCI = dict()\npop_stdev = population_ages.std()  # Get the population standard deviation\n\nmargin_of_error = z_critical * (pop_stdev/math.sqrt(sample_size))\n\nconfidence_interval = (sample_mean - margin_of_error,\n                       sample_mean + margin_of_error)  \n\n#sample_mean =  sample_mean*100\n\nprint('Confidence interval: ({0:.4f}, {1:.4f})'.format(confidence_interval[0], confidence_interval[1]))\nprint('sample_mean: {0:5.4f}'.format(sample_mean))\nprint('margin_of_error:{0:5.4f}'.format(margin_of_error))\n\nCI[\"pop_stdev\",\"margin_of_error\",\"confidence_interval\"]=pop_stdev, margin_of_error, confidence_interval\n\nwith open(\"CI.txt\", \"wb\") as fp:   #Pickling\n    pickle.dump(CI, fp)","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-05-21T16:08:17.892720Z","iopub.execute_input":"2021-05-21T16:08:17.893659Z","iopub.status.idle":"2021-05-21T16:08:17.913248Z","shell.execute_reply.started":"2021-05-21T16:08:17.893612Z","shell.execute_reply":"2021-05-21T16:08:17.911382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom imblearn.metrics import sensitivity_specificity_support\nfrom sklearn.metrics import classification_report\nimport seaborn as sn\nimport pandas as pd\n#model = load_model('EfN_model-1.h5')\n#test_generator.reset()\n#pred = model.predict_generator(test_generator, steps=None, callbacks=None, max_queue_size=10, workers=1, use_multiprocessing=False, verbose=0)\n#pred_label=np.argmax(pred,axis=1)\n#true_label = test_generator.classes\n#pred_label=yhat\n#true_label = testy1 \ntrue_label, pred_label = true_label, mc_ensemble_pred\npred = np.array(mc_predictions).mean(axis=0)\n\n#loss, acc = model.evaluate_generator(test_generator,steps=None)\n\nacc = accuracy_score(true_label, pred_label)\n\n\ntarget_names = ['Normal', 'Mild','Moderate', 'Proliferative', 'Severe']\n\nresult = sensitivity_specificity_support(true_label, pred_label, average='macro')\n\nprint(\"Sensitivity: {:5.2f}%\".format(100*result[0]), \"specificity {:5.2f}%\".format(100*result[1]), \n      \"Accuracy: {:5.2f}%\".format(100*acc),'\\n')\n\nreport=classification_report(true_label, pred_label, target_names=target_names, digits=4)\nprint(report)\n\nf = open( 'report.txt', 'w' )\nf.write(report)\nf.close()\n\n\nmatrix = confusion_matrix(true_label, pred_label)\nmatrix = matrix.astype('float')\n#cm_norm = matrix / matrix.sum(axis=1)[:, np.newaxis]\nprint(matrix)\n#class_acc = np.array(cm_norm.diagonal())\nclass_acc = [matrix[i,i]/np.sum(matrix[i,:]) if np.sum(matrix[i,:]) else 0 for i in range(len(matrix))]\nprint('Sens Normal: {0:.3f}, Mild: {1:.3f}, Moderate: {2:.3f}, Severe: {3:.3f}, Proliferative: {4:.3f}'.format(class_acc[0], class_acc[1], class_acc[2], class_acc[3], class_acc[4]))\n                                                                               \nppvs = [matrix[i,i]/np.sum(matrix[:,i]) if np.sum(matrix[:,i]) else 0 for i in range(len(matrix))]\nprint('PPV Normal: {0:.3f}, Mild: {1:.3f}, Moderate: {2:.3f}, Severe: {3:.3f}, Proliferative: {4:.3f}'.format(ppvs[0],  ppvs[1], ppvs[2],  ppvs[3], ppvs[4]))\nM = ((ppvs[0]+ppvs[1]+ppvs[2]+ppvs[3]+ppvs[4])/5)*100\nprint('Mean PPV:{0:.2f}'.format(M))  \n\ndisp = confusion_matrix(true_label, pred_label)\ndisp.astype('int')\npd.options.display.float_format = '{:.5f}'.format\ndf_cm = pd.DataFrame(disp, target_names, target_names)\n# plt.figure(figsize=(10,7))\nfig, ax = plt.subplots(figsize=(5,4))\nsn.set(font_scale=1.5) # for label size\nsn.heatmap(df_cm, annot=True, annot_kws={\"size\": 15},ax=ax, cmap=\"YlOrBr\" , fmt='g',cbar=False) # font size\nplt.ylabel('Actual',fontsize=20)\nplt.xlabel('Predicted',fontsize=20)\nplt.ioff()\nplt.savefig(\"confusion_mat\", bbox_inches='tight')\nplt.show()\n#plt.savefig('Confusion_mat.png')","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-05-21T16:08:17.915600Z","iopub.execute_input":"2021-05-21T16:08:17.916591Z","iopub.status.idle":"2021-05-21T16:08:18.715240Z","shell.execute_reply.started":"2021-05-21T16:08:17.916543Z","shell.execute_reply":"2021-05-21T16:08:18.714188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_ci_auc( sample_size, accs):\n    import scipy.stats as stats\n    import math\n\n    #np.random.seed(10)\n    #population_ages = accs\n    #sample_size = 200\n    population_ages= sample = np.asarray(accs, dtype=np.float32)\n    sample_mean = sample.mean()\n\n    z_critical = stats.norm.ppf(q = 0.995)  # Get the z-critical value*\n\n    #z_critical=1.96\n    print(\"z-critical value:\",z_critical)              # Check the z-critical value\n\n\n    pop_stdev = population_ages.std()  # Get the population standard deviation\n\n    margin_of_error = z_critical * (pop_stdev/math.sqrt(sample_size))\n\n    confidence_interval = (sample_mean - margin_of_error,\n                           sample_mean + margin_of_error)  \n\n    #sample_mean =  sample_mean*100\n\n    print('Confidence interval: ({0:.6f}, {1:.6f})'.format(confidence_interval[0], confidence_interval[1]))\n    print('sample_mean: {0:5.6f}'.format(sample_mean))\n    print('margin_of_error:{0:5.6f}'.format(margin_of_error))\n    \n    return confidence_interval, margin_of_error, sample_mean\n\n\n\n##############################\n\nprint(__doc__)\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom itertools import cycle\n\nfrom sklearn import svm, datasets\nfrom sklearn.metrics import roc_curve, auc\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import label_binarize\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom scipy import interp\nfrom sklearn.metrics import roc_auc_score\nfrom keras.utils import to_categorical\n\ntrue_label2 = to_categorical(true_label)\n#pred_label = yscore\n# Compute ROC curve and ROC area for each class\nfpr = dict()\ntpr = dict()\nyscore=pred\nroc_auc = dict()\nn_classes = 5\n\nC=np.zeros(len(accs))\nN=np.zeros(len(accs))\nP=np.zeros(len(accs))\nK=np.zeros(len(accs))\nL=np.zeros(len(accs))\n\nAv=np.zeros(len(accs))\nfor j in range(len(accs)):\n    yscore=mc_predictions[j]\n    i=0\n    fpr[i], tpr[i], _ = roc_curve(true_label2[:, i], yscore[:, i])\n    C[j]= auc(fpr[i], tpr[i])\n    i=1\n    fpr[i], tpr[i], _ = roc_curve(true_label2[:, i], yscore[:, i])\n    N[j]= auc(fpr[i], tpr[i])\n    i=2\n    fpr[i], tpr[i], _ = roc_curve(true_label2[:, i], yscore[:, i])\n    P[j]= auc(fpr[i], tpr[i])\n    i=3\n    fpr[i], tpr[i], _ = roc_curve(true_label2[:, i], yscore[:, i])\n    K[j]= auc(fpr[i], tpr[i])\n    i=4\n    fpr[i], tpr[i], _ = roc_curve(true_label2[:, i], yscore[:, i])\n    L[j]= auc(fpr[i], tpr[i])\n        \n    fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(true_label2.ravel(), yscore.ravel())\n    Av[j] = auc(fpr[\"micro\"], tpr[\"micro\"])\n       \n   \n\n# AUC, margin_of_error and CI    \n\nsample_size=len(accs)\ncl,M0, CI0 = get_ci_auc(sample_size,C)\ncl,M1, CI1 = get_ci_auc(sample_size,N) \ncl,M2, CI2 = get_ci_auc(sample_size,P)\ncl,M3, CI3 = get_ci_auc(sample_size,K)\ncl,M4, CI4 = get_ci_auc(sample_size,L) \ncl,Mav,CIav = get_ci_auc(sample_size,Av) \n\n\nimport scipy.io as sio\nsio.savemat('fpr.mat',{'fpr':fpr});\n\n'''\nsorted_scores = np.array(C)\nsorted_scores.sort()\n\n\n# 95% c.i.\nconfidence_lower = sorted_scores[int(0.05 * len(sorted_scores))]\nconfidence_upper = sorted_scores[int(0.95 * len(sorted_scores))] \nM = (confidence_lower+confidence_upper)/2 \n'''    \n\n\n#fpr[\"macro\"] = all_fpr\n#tpr[\"macro\"] = mean_tpr\n#roc_auc[\"macro\"] = auc(fpr[\"macro\"], tpr[\"macro\"])\n\n# Plot all ROC curves\nlw = 1.5\nplt.figure()\nplt.plot(fpr[\"micro\"], tpr[\"micro\"],color='k',linestyle=':', linewidth=1.5,\n         label='Micro-Avg(AUC=[{:0.2f}±{:0.4f}]'. format(CIav,Mav))\n\n\nplt.plot(fpr[0], tpr[0], color='b', \n         lw=lw, label='Normal(AUC=[{:0.2f}±{:0.4f}]'. format(CI0,M0))\n\n\nplt.plot(fpr[1], tpr[1], color='g',\n         lw=lw, label='Mild(AUC=[{:0.2f}±{:0.4f}]'. format(CI1,M1))\n\nplt.plot(fpr[2], tpr[2], color='r', \n        lw=lw, label='Moderate(AUC=[{:0.2f}±{:0.4f}]'. format(CI2,M2))\n\nplt.plot(fpr[3], tpr[3], color='y', \n         lw=lw, label='Proliferative(AUC=[{:0.2f}±{:0.4f}]'. format(CI3,M3))\n\nplt.plot(fpr[4], tpr[4], color='c',\n        lw=lw, label='Severe(AUC=[{:0.2f}±{:0.4f}]'. format(CI4,M4))\n\n\n\nimport matplotlib \n#plt.style.use('seaborn-whitegrid')\n\nsize=14\nparams = {'legend.fontsize': 10,\n          'figure.figsize': (4,3),\n          'axes.labelsize': size,\n          'axes.titlesize': size,\n          'xtick.labelsize': size*0.8,\n          'ytick.labelsize': size*0.8,\n          'lines.linewidth':5,\n          'axes.titlepad': 5}\nplt.rcParams.update(params)\n\n#fig, ax = plt.subplots()\nax = plt.axes()\n\n# Don't allow the axis to be on top of your data\nax.set_axisbelow(False)\n\n# Customize the grid\nax.grid(False)\n\n# Setting the background color\nax.set_facecolor(\"white\")\n#legend = ax.legend(loc='lower center', shadow=True, fontsize='x-large')\n#legend.get_frame().set_facecolor('#00FFCC')\n\n#plt.style.use('classic')\n#plt.rc('lines', linewidth=4)\n#plt.legend.fontsize = 10\n#plt.plot([0, 1], [0, 1], 'k--', lw=lw)\nplt.xlim([0.0, 1.0])\nplt.ylim(top=1.001)\nplt.ylim(bottom=.1)\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.legend(loc=\"lower right\")\n#plt.rcParams[\"figure.figsize\"] = (6,4)\n#plt.grid(color='lightgrey', linestyle='-')\n#plt.savefig(\"Bfold5dist.eps\",format = 'eps',bbox_inches='tight')\nplt.savefig('BfoldROC.eps', format = 'eps',bbox_inches='tight')\n\n#plt.savefig(\"foldROC\", bbox_inches='tight')\nplt.show()","metadata":{"editable":false,"execution":{"iopub.status.busy":"2021-05-21T16:08:18.720282Z","iopub.execute_input":"2021-05-21T16:08:18.722948Z","iopub.status.idle":"2021-05-21T16:08:19.500487Z","shell.execute_reply.started":"2021-05-21T16:08:18.722901Z","shell.execute_reply":"2021-05-21T16:08:19.499280Z"},"trusted":true},"execution_count":null,"outputs":[]}]}