{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":4104,"databundleVersionId":46661,"sourceType":"competition"},{"sourceId":7251,"sourceType":"datasetVersion","datasetId":2798},{"sourceId":164933,"sourceType":"modelInstanceVersion","modelInstanceId":140313,"modelId":162939}],"dockerImageVersionId":335,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# copy the weights and configurations for the pre-trained models\n!mkdir ~/.keras\n!mkdir ~/.keras/models\n!cp ../input/keras-pretrained-models/*notop* ~/.keras/models/\n!cp ../input/keras-pretrained-models/imagenet_class_index.json ~/.keras/models/","metadata":{"_uuid":"c163e45042a69905855f7c04a65676e5aca4837b","_cell_guid":"e94de3e7-de1f-4c18-ad1f-c8b686127340","execution":{"iopub.status.busy":"2024-11-16T17:53:27.533572Z","iopub.execute_input":"2024-11-16T17:53:27.533897Z","iopub.status.idle":"2024-11-16T17:53:37.091336Z","shell.execute_reply.started":"2024-11-16T17:53:27.533835Z","shell.execute_reply":"2024-11-16T17:53:37.090399Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls ~/.keras/models","metadata":{"execution":{"iopub.status.busy":"2024-11-16T17:53:37.092626Z","iopub.execute_input":"2024-11-16T17:53:37.09287Z","iopub.status.idle":"2024-11-16T17:53:38.115498Z","shell.execute_reply.started":"2024-11-16T17:53:37.092834Z","shell.execute_reply":"2024-11-16T17:53:38.114954Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls data/train_11\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-16T17:53:38.117027Z","iopub.execute_input":"2024-11-16T17:53:38.117389Z","iopub.status.idle":"2024-11-16T17:53:39.109298Z","shell.execute_reply.started":"2024-11-16T17:53:38.117326Z","shell.execute_reply":"2024-11-16T17:53:39.108628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!unzip ../input/diabetic-retinopathy-detection/trainLabels.csv.zip\n!apt install p7zip-full -y\n!7z x ../input/diabetic-retinopathy-detection/train.zip.001 \"-i!train/11*.jpeg\" -y # restrict extracted file to about 100 for the disk restriction\n!mkdir data\n!mv train data/train_11","metadata":{"execution":{"iopub.status.busy":"2024-11-16T17:53:39.11101Z","iopub.execute_input":"2024-11-16T17:53:39.111391Z","iopub.status.idle":"2024-11-16T17:54:09.817024Z","shell.execute_reply.started":"2024-11-16T17:53:39.111317Z","shell.execute_reply":"2024-11-16T17:54:09.816098Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # showing and rendering figures\n# io related\nfrom skimage.io import imread\nimport os\nfrom glob import glob\n# not needed in Kaggle, but required in Jupyter\n%matplotlib inline ","metadata":{"_uuid":"725d378daf5f836d4885d67240fc7955f113309d","_cell_guid":"c3cc4285-bfa4-4612-ac5f-13d10678c09a","execution":{"iopub.status.busy":"2024-11-16T17:54:09.818853Z","iopub.execute_input":"2024-11-16T17:54:09.819314Z","iopub.status.idle":"2024-11-16T17:54:10.256211Z","shell.execute_reply.started":"2024-11-16T17:54:09.81923Z","shell.execute_reply":"2024-11-16T17:54:10.255748Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_image_dir = os.path.join('/', 'kaggle', 'working', 'data', 'train_11')\nretina_df = pd.read_csv('/kaggle/working/trainLabels.csv')\nretina_df['PatientId'] = retina_df['image'].map(lambda x: x.split('_')[0])\nretina_df['path'] = retina_df['image'].map(lambda x: os.path.join(base_image_dir,\n                                                         '{}.jpeg'.format(x)))\nretina_df['exists'] = retina_df['path'].map(os.path.exists)\nprint(retina_df['exists'].sum(), 'images found of', retina_df.shape[0], 'total')\nretina_df['eye'] = retina_df['image'].map(lambda x: 1 if x.split('_')[-1]=='left' else 0)\nfrom keras.utils.np_utils import to_categorical\nretina_df['level_cat'] = retina_df['level'].map(lambda x: to_categorical(x, 1+retina_df['level'].max()))\n\nretina_df.dropna(inplace = True)\nretina_df = retina_df[retina_df['exists']]\nretina_df.sample(3)","metadata":{"_uuid":"346da81db6ee7a34af8da8af245b42e681f2ba48","_cell_guid":"c4b38df6-ffa1-4847-b605-511e72b68231","execution":{"iopub.status.busy":"2024-11-16T17:54:10.257322Z","iopub.execute_input":"2024-11-16T17:54:10.257543Z","iopub.status.idle":"2024-11-16T17:54:20.115874Z","shell.execute_reply.started":"2024-11-16T17:54:10.257504Z","shell.execute_reply":"2024-11-16T17:54:20.115129Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"retina_df[['level', 'eye']].hist(figsize = (10, 5))","metadata":{"_uuid":"60a8111c4093ca6f69d27a4499442ba7dd750839","_cell_guid":"5c8bd288-8261-4cbe-a954-e62ac795cc3e","execution":{"iopub.status.busy":"2024-11-16T17:54:20.117189Z","iopub.execute_input":"2024-11-16T17:54:20.117495Z","iopub.status.idle":"2024-11-16T17:54:20.387553Z","shell.execute_reply.started":"2024-11-16T17:54:20.117431Z","shell.execute_reply":"2024-11-16T17:54:20.386941Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nrr_df = retina_df[['PatientId', 'level']].drop_duplicates()\ntrain_ids, valid_ids = train_test_split(rr_df['PatientId'], \n                                   test_size = 0.25, \n                                   random_state = 2018,\n                                   stratify = rr_df['level'])\nraw_train_df = retina_df[retina_df['PatientId'].isin(train_ids)]\nvalid_df = retina_df[retina_df['PatientId'].isin(valid_ids)]\nprint('train', raw_train_df.shape[0], 'validation', valid_df.shape[0])","metadata":{"_uuid":"a48b300ca4d37a6e8b39f82e3c172739635e4baa","_cell_guid":"1192c6b3-a940-4fa0-a498-d7e0d400a796","execution":{"iopub.status.busy":"2024-11-16T17:54:20.388745Z","iopub.execute_input":"2024-11-16T17:54:20.389087Z","iopub.status.idle":"2024-11-16T17:54:20.65345Z","shell.execute_reply.started":"2024-11-16T17:54:20.389003Z","shell.execute_reply":"2024-11-16T17:54:20.65293Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = raw_train_df.groupby(['level', 'eye']).apply(lambda x: x.sample(75, replace = True)\n                                                      ).reset_index(drop = True)\nprint('New Data Size:', train_df.shape[0], 'Old Size:', raw_train_df.shape[0])\ntrain_df[['level', 'eye']].hist(figsize = (10, 5))","metadata":{"_uuid":"ba7befa238b8c11f9672e3539ac58f3da6955bd9","_cell_guid":"7a130199-fbf6-4c60-95f5-0797b2f3eaf1","execution":{"iopub.status.busy":"2024-11-16T17:54:20.654583Z","iopub.execute_input":"2024-11-16T17:54:20.654787Z","iopub.status.idle":"2024-11-16T17:54:20.966065Z","shell.execute_reply.started":"2024-11-16T17:54:20.65475Z","shell.execute_reply":"2024-11-16T17:54:20.965446Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nfrom keras import backend as K\nfrom keras.applications.inception_v3 import preprocess_input\nimport numpy as np\nIMG_SIZE = (512, 512) # slightly smaller than vgg16 normally expects\ndef tf_image_loader(out_size, \n                      horizontal_flip = True, \n                      vertical_flip = False, \n                     random_brightness = True,\n                     random_contrast = True,\n                    random_saturation = True,\n                    random_hue = True,\n                      color_mode = 'rgb',\n                       preproc_func = preprocess_input,\n                       on_batch = False):\n    def _func(X):\n        with tf.name_scope('image_augmentation'):\n            with tf.name_scope('input'):\n                X = tf.image.decode_png(tf.read_file(X), channels = 3 if color_mode == 'rgb' else 0)\n                X = tf.image.resize_images(X, out_size)\n            with tf.name_scope('augmentation'):\n                if horizontal_flip:\n                    X = tf.image.random_flip_left_right(X)\n                if vertical_flip:\n                    X = tf.image.random_flip_up_down(X)\n                if random_brightness:\n                    X = tf.image.random_brightness(X, max_delta = 0.1)\n                if random_saturation:\n                    X = tf.image.random_saturation(X, lower = 0.75, upper = 1.5)\n                if random_hue:\n                    X = tf.image.random_hue(X, max_delta = 0.15)\n                if random_contrast:\n                    X = tf.image.random_contrast(X, lower = 0.75, upper = 1.5)\n                return preproc_func(X)\n    if on_batch: \n        # we are meant to use it on a batch\n        def _batch_func(X, y):\n            return tf.map_fn(_func, X), y\n        return _batch_func\n    else:\n        # we apply it to everything\n        def _all_func(X, y):\n            return _func(X), y         \n        return _all_func\n    \ndef tf_augmentor(out_size,\n                intermediate_size = (640, 640),\n                 intermediate_trans = 'crop',\n                 batch_size = 16,\n                   horizontal_flip = True, \n                  vertical_flip = False, \n                 random_brightness = True,\n                 random_contrast = True,\n                 random_saturation = True,\n                    random_hue = True,\n                  color_mode = 'rgb',\n                   preproc_func = preprocess_input,\n                   min_crop_percent = 0.001,\n                   max_crop_percent = 0.005,\n                   crop_probability = 0.5,\n                   rotation_range = 10):\n    \n    load_ops = tf_image_loader(out_size = intermediate_size, \n                               horizontal_flip=horizontal_flip, \n                               vertical_flip=vertical_flip, \n                               random_brightness = random_brightness,\n                               random_contrast = random_contrast,\n                               random_saturation = random_saturation,\n                               random_hue = random_hue,\n                               color_mode = color_mode,\n                               preproc_func = preproc_func,\n                               on_batch=False)\n    def batch_ops(X, y):\n        batch_size = tf.shape(X)[0]\n        with tf.name_scope('transformation'):\n            # code borrowed from https://becominghuman.ai/data-augmentation-on-gpu-in-tensorflow-13d14ecf2b19\n            # The list of affine transformations that our image will go under.\n            # Every element is Nx8 tensor, where N is a batch size.\n            transforms = []\n            identity = tf.constant([1, 0, 0, 0, 1, 0, 0, 0], dtype=tf.float32)\n            if rotation_range > 0:\n                angle_rad = rotation_range / 180 * np.pi\n                angles = tf.random_uniform([batch_size], -angle_rad, angle_rad)\n                transforms += [tf.contrib.image.angles_to_projective_transforms(angles, intermediate_size[0], intermediate_size[1])]\n\n            if crop_probability > 0:\n                crop_pct = tf.random_uniform([batch_size], min_crop_percent, max_crop_percent)\n                left = tf.random_uniform([batch_size], 0, intermediate_size[0] * (1.0 - crop_pct))\n                top = tf.random_uniform([batch_size], 0, intermediate_size[1] * (1.0 - crop_pct))\n                crop_transform = tf.stack([\n                      crop_pct,\n                      tf.zeros([batch_size]), top,\n                      tf.zeros([batch_size]), crop_pct, left,\n                      tf.zeros([batch_size]),\n                      tf.zeros([batch_size])\n                  ], 1)\n                coin = tf.less(tf.random_uniform([batch_size], 0, 1.0), crop_probability)\n                transforms += [tf.where(coin, crop_transform, tf.tile(tf.expand_dims(identity, 0), [batch_size, 1]))]\n            if len(transforms)>0:\n                X = tf.contrib.image.transform(X,\n                      tf.contrib.image.compose_transforms(*transforms),\n                      interpolation='BILINEAR') # or 'NEAREST'\n            if intermediate_trans=='scale':\n                X = tf.image.resize_images(X, out_size)\n            elif intermediate_trans=='crop':\n                X = tf.image.resize_image_with_crop_or_pad(X, out_size[0], out_size[1])\n            else:\n                raise ValueError('Invalid Operation {}'.format(intermediate_trans))\n            return X, y\n    def _create_pipeline(in_ds):\n        batch_ds = in_ds.map(load_ops, num_parallel_calls=4).batch(batch_size)\n        return batch_ds.map(batch_ops)\n    return _create_pipeline","metadata":{"_uuid":"9529ab766763a9f122786464c24ab1ebe22c6006","_cell_guid":"9954bfda-29bd-4c4d-b526-0a972b3e43e2","execution":{"iopub.status.busy":"2024-11-16T17:54:20.967304Z","iopub.execute_input":"2024-11-16T17:54:20.967531Z","iopub.status.idle":"2024-11-16T17:54:21.327647Z","shell.execute_reply.started":"2024-11-16T17:54:20.967492Z","shell.execute_reply":"2024-11-16T17:54:21.326939Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def flow_from_dataframe(idg, \n                        in_df, \n                        path_col,\n                        y_col, \n                        shuffle = True, \n                        color_mode = 'rgb'):\n    files_ds = tf.data.Dataset.from_tensor_slices((in_df[path_col].values, \n                                                   np.stack(in_df[y_col].values,0)))\n    in_len = in_df[path_col].values.shape[0]\n    while True:\n        if shuffle:\n            files_ds = files_ds.shuffle(in_len) # shuffle the whole dataset\n        \n        next_batch = idg(files_ds).repeat().make_one_shot_iterator().get_next()\n        for i in range(max(in_len//32,1)):\n            # NOTE: if we loop here it is 'thread-safe-ish' if we loop on the outside it is completely unsafe\n            yield K.get_session().run(next_batch)","metadata":{"_uuid":"07851e798db3d89ba13db7d4b56ab2b759221464","_cell_guid":"b5767f42-da63-4737-8f50-749c1a25aa84","execution":{"iopub.status.busy":"2024-11-16T17:54:21.328824Z","iopub.execute_input":"2024-11-16T17:54:21.329055Z","iopub.status.idle":"2024-11-16T17:54:21.34675Z","shell.execute_reply.started":"2024-11-16T17:54:21.329009Z","shell.execute_reply":"2024-11-16T17:54:21.346246Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"batch_size = 48\ncore_idg = tf_augmentor(out_size = IMG_SIZE, \n                        color_mode = 'rgb', \n                        vertical_flip = True,\n                        crop_probability=0.0, # crop doesn't work yet\n                        batch_size = batch_size) \nvalid_idg = tf_augmentor(out_size = IMG_SIZE, color_mode = 'rgb', \n                         crop_probability=0.0, \n                         horizontal_flip = False, \n                         vertical_flip = False, \n                         random_brightness = False,\n                         random_contrast = False,\n                         random_saturation = False,\n                         random_hue = False,\n                         rotation_range = 0,\n                        batch_size = batch_size)\n\ntrain_gen = flow_from_dataframe(core_idg, train_df, \n                             path_col = 'path',\n                            y_col = 'level_cat')\n\nvalid_gen = flow_from_dataframe(valid_idg, valid_df, \n                             path_col = 'path',\n                            y_col = 'level_cat') # we can use much larger batches for evaluation","metadata":{"_uuid":"1848f5048a9e00668c3778a85deea97f980e4f1c","_cell_guid":"810bd229-fec9-43c4-b3bd-afd62e3e9552","execution":{"iopub.status.busy":"2024-11-16T17:54:21.347853Z","iopub.execute_input":"2024-11-16T17:54:21.348158Z","iopub.status.idle":"2024-11-16T17:54:21.362519Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"t_x, t_y = next(valid_gen)\nfig, m_axs = plt.subplots(2, 4, figsize = (16, 8))\nfor (c_x, c_y, c_ax) in zip(t_x, t_y, m_axs.flatten()):\n    c_ax.imshow(np.clip(c_x*127+127, 0, 255).astype(np.uint8))\n    c_ax.set_title('Severity {}'.format(np.argmax(c_y, -1)))\n    c_ax.axis('off')","metadata":{"_uuid":"6810407e25b887dd8b352f1e46fb3faceaa58ab7","execution":{"iopub.status.busy":"2024-11-16T17:54:21.363586Z","iopub.execute_input":"2024-11-16T17:54:21.363866Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"t_x, t_y = next(train_gen)\nfig, m_axs = plt.subplots(2, 4, figsize = (16, 8))\nfor (c_x, c_y, c_ax) in zip(t_x, t_y, m_axs.flatten()):\n    c_ax.imshow(np.clip(c_x*127+127, 0, 255).astype(np.uint8))\n    c_ax.set_title('Severity {}'.format(np.argmax(c_y, -1)))\n    c_ax.axis('off')","metadata":{"_uuid":"8190b4ad60d49fa65af074dd138a19cb8787e983","scrolled":true,"_cell_guid":"2d62234f-aeb0-4eba-8a38-d713d819abf6","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.applications.resnet50 import ResNet50 as PTModel\n\nfrom keras.layers import GlobalAveragePooling2D, Dense, Dropout, Flatten, Input, Conv2D, multiply, LocallyConnected2D, Lambda\nfrom keras.models import Model\nin_lay = Input(t_x.shape[1:])\nbase_pretrained_model = PTModel(input_shape =  t_x.shape[1:], include_top = False, weights = 'imagenet')\nbase_pretrained_model.trainable = False\npt_depth = base_pretrained_model.get_output_shape_at(0)[-1]\npt_features = base_pretrained_model(in_lay)\nfrom keras.layers import BatchNormalization\nbn_features = BatchNormalization()(pt_features)\n\n# here we do an attention mechanism to turn pixels in the GAP on an off\n\nattn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\nattn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\nattn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\nattn_layer = Conv2D(1, \n                    kernel_size = (1,1), \n                    padding = 'valid', \n                    activation = 'sigmoid')(attn_layer)\n# fan it out to all of the channels\nup_c2_w = np.ones((1, 1, 1, pt_depth))\nup_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', \n               activation = 'linear', use_bias = False, weights = [up_c2_w])\nup_c2.trainable = False\nattn_layer = up_c2(attn_layer)\n\nmask_features = multiply([attn_layer, bn_features])\ngap_features = GlobalAveragePooling2D()(mask_features)\ngap_mask = GlobalAveragePooling2D()(attn_layer)\n# to account for missing values from the attention model\ngap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\ngap_dr = Dropout(0.25)(gap)\ndr_steps = Dropout(0.25)(Dense(128, activation = 'relu')(gap_dr))\nout_layer = Dense(t_y.shape[-1], activation = 'softmax')(dr_steps)\nretina_model = Model(inputs = [in_lay], outputs = [out_layer])\nfrom keras.metrics import top_k_categorical_accuracy\ndef top_2_accuracy(in_gt, in_pred):\n    return top_k_categorical_accuracy(in_gt, in_pred, k=2)\n\nretina_model.compile(optimizer = 'adam', loss = 'categorical_crossentropy',\n                           metrics = ['categorical_accuracy', top_2_accuracy])\nretina_model.summary()","metadata":{"_uuid":"1f0dfaccda346d7bc4758e7329d61028d254a8d6","_cell_guid":"eeb36110-0cde-4450-a43c-b8f707adb235","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau\nweight_path=\"{}_weights.best.hdf5\".format('retina')\n\ncheckpoint = ModelCheckpoint(weight_path, monitor='val_loss', verbose=1, \n                             save_best_only=True, mode='min', save_weights_only = True)\n\nreduceLROnPlat = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=3, verbose=1, mode='auto', epsilon=0.0001, cooldown=5, min_lr=0.0001)\nearly = EarlyStopping(monitor=\"val_loss\", \n                      mode=\"min\", \n                      patience=6) # probably needs to be more patient, but kaggle time is limited\ncallbacks_list = [checkpoint, early, reduceLROnPlat]","metadata":{"_uuid":"48b9764e16fb5af52aed35c82bae6299e67d5bc7","_cell_guid":"17803ae1-bed8-41a4-9a2c-e66287a24830","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!rm -rf ~/.keras # clean up before starting training","metadata":{"_uuid":"78dfa383c51777377c1f81e42017cbcca5f5736f","_cell_guid":"84f7cdec-ca00-460c-9991-55b1f7f02f20","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"retina_model.fit_generator(train_gen, \n                           steps_per_epoch = train_df.shape[0]//batch_size,\n                           validation_data = valid_gen, \n                           validation_steps = valid_df.shape[0]//batch_size,\n                              epochs = 2, \n                              callbacks = callbacks_list,\n                             workers = 0, # tf-generators are not thread-safe\n                             use_multiprocessing=False, \n                             max_queue_size = 0\n                            )","metadata":{"_uuid":"b2148479bfe41c5d9fd0faece4c75adea509dabe","_cell_guid":"58a75586-442b-4804-84a6-d63d5a42ea14","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# load the best version of the model\nretina_model.load_weights(weight_path)\nretina_model.save('full_retina_model.h5')","metadata":{"_uuid":"3a90f05dd206cd76c72d8c6278ebb93da41ee45f","_cell_guid":"4d0c45b0-bb23-48d2-83eb-bc3990043e26","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##### create one fixed dataset for evaluating\nfrom tqdm import tqdm_notebook\n# fresh valid gen\nvalid_gen = flow_from_dataframe(valid_idg, valid_df, \n                             path_col = 'path',\n                            y_col = 'level_cat') \nvbatch_count = (valid_df.shape[0]//batch_size-1)\nout_size = vbatch_count*batch_size\ntest_X = np.zeros((out_size,)+t_x.shape[1:], dtype = np.float32)\ntest_Y = np.zeros((out_size,)+t_y.shape[1:], dtype = np.float32)\nfor i, (c_x, c_y) in zip(tqdm_notebook(range(vbatch_count)), \n                         valid_gen):\n    j = i*batch_size\n    test_X[j:(j+c_x.shape[0])] = c_x\n    test_Y[j:(j+c_x.shape[0])] = c_y","metadata":{"_uuid":"2b74f4ab850c6e82549d732b6f0524724b95b53c","_cell_guid":"f37dd4d8-ecd6-487a-90d8-74fe14a9a318","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# get the attention layer since it is the only one with a single output dim\nfor attn_layer in retina_model.layers:\n    c_shape = attn_layer.get_output_shape_at(0)\n    if len(c_shape)==4:\n        if c_shape[-1]==1:\n            print(attn_layer)\n            break","metadata":{"_uuid":"ad5b085d351e79b950bf0c2ddc476799d5b0692f","_cell_guid":"e41a063f-35c9-410f-be63-f66b63ff9683","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import keras.backend as K\nrand_idx = np.random.choice(range(len(test_X)), size = 6)\nattn_func = K.function(inputs = [retina_model.get_input_at(0), K.learning_phase()],\n           outputs = [attn_layer.get_output_at(0)]\n          )\nfig, m_axs = plt.subplots(len(rand_idx), 2, figsize = (8, 4*len(rand_idx)))\n[c_ax.axis('off') for c_ax in m_axs.flatten()]\nfor c_idx, (img_ax, attn_ax) in zip(rand_idx, m_axs):\n    cur_img = test_X[c_idx:(c_idx+1)]\n    attn_img = attn_func([cur_img, 0])[0]\n    img_ax.imshow(np.clip(cur_img[0,:,:,:]*127+127, 0, 255).astype(np.uint8))\n    attn_ax.imshow(attn_img[0, :, :, 0]/attn_img[0, :, :, 0].max(), cmap = 'viridis', \n                   vmin = 0, vmax = 1, \n                   interpolation = 'lanczos')\n    real_cat = np.argmax(test_Y[c_idx, :])\n    img_ax.set_title('Eye Image\\nCat:%2d' % (real_cat))\n    pred_cat = retina_model.predict(cur_img)\n    attn_ax.set_title('Attention Map\\nPred:%2.2f%%' % (100*pred_cat[0,real_cat]))\nfig.savefig('attention_map.png', dpi = 300)","metadata":{"_uuid":"00850972ae4298f49ed1838b3fc49c2d8fb07547","_cell_guid":"340eef36-f5b2-4b15-a59f-440061a427eb","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nfrom torchvision import models, transforms\nimport matplotlib.pyplot as plt\n\n# List of image paths\nimage_paths = [\n    '/kaggle/working/data/train_11/11779_right.jpeg', '/kaggle/working/data/train_11/1177_left.jpeg',\n    '/kaggle/working/data/train_11/1177_right.jpeg', '/kaggle/working/data/train_11/11780_left.jpeg',\n    '/kaggle/working/data/train_11/11780_right.jpeg', '/kaggle/working/data/train_11/11781_left.jpeg',\n    '/kaggle/working/data/train_11/11786_left.jpeg',\n    '/kaggle/working/data/train_11/11786_right.jpeg', '/kaggle/working/data/train_11/11789_left.jpeg',\n    '/kaggle/working/data/train_11/11789_right.jpeg', \n    '/kaggle/working/data/train_11/1178_right.jpeg'\n]\n\n# Step 2: Preprocess the Image\npreprocess = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n# Step 3: Load Pre-trained Model and Set to Evaluation Mode\nmodel = models.resnet50(pretrained=True)\nmodel.eval()\n\n# Step 4: Define Grad-CAM Function\nclass GradCAM:\n    def __init__(self, model, target_layer):\n        self.model = model\n        self.target_layer = target_layer\n        self.gradients = None\n        self.activations = None\n        self.hook_layers()\n\n    def hook_layers(self):\n        def forward_hook(module, input, output):\n            self.activations = output.detach()\n\n        def backward_hook(module, grad_in, grad_out):\n            self.gradients = grad_out[0].detach()\n\n        target_layer = dict(self.model.named_modules())[self.target_layer]\n        target_layer.register_forward_hook(forward_hook)\n        target_layer.register_backward_hook(backward_hook)\n\n    def generate_heatmap(self, input_tensor, class_index=None):\n        # Forward pass\n        output = self.model(input_tensor)\n\n        # Convert output to Tensor if it's not already\n        if isinstance(output, torch.Tensor):\n            output = output\n        else:\n            output = output.data  # Handle old versions of PyTorch\n\n        if class_index is None:\n            # Ensure output is a tensor and use max(1) to get the class index\n            class_index = output.max(1)[1].item()\n\n        score = output[:, class_index]\n\n        # Backward pass\n        self.model.zero_grad()\n        score.backward()\n\n        # Calculate Grad-CAM\n        weights = self.gradients.mean(dim=2, keepdim=True)  # Fix: Use dim=2 for channel dimension\n        cam = (weights * self.activations).sum(dim=1)[0]  # Summing across channels (dim=1)\n        cam = F.relu(cam)  # Apply ReLU to keep positive values\n        cam = cam / cam.max()  # Normalize\n\n        # Resize to match original image size\n        cam = cv2.resize(cam.cpu().numpy(), (224, 224))\n        return cam\n\n# Initialize GradCAM with the target layer\ngrad_cam = GradCAM(model=model, target_layer='layer4')\n\n# Step 5: Loop through each image and display results\nfig, axs = plt.subplots(len(image_paths), 2, figsize=(10, 5 * len(image_paths)))\n\nfor i, image_path in enumerate(image_paths):\n    # Load and preprocess each image\n    image = cv2.imread(image_path)\n    if image is None:\n        print(f\"Image not found at path: {image_path}\")\n        continue\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    original_image = image.copy()\n    image = cv2.resize(image, (224, 224))\n    input_tensor = preprocess(image).unsqueeze(0)  # Add batch dimension\n\n    # Generate Grad-CAM heatmap\n    heatmap = grad_cam.generate_heatmap(input_tensor)\n    heatmap = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)\n    overlay = cv2.addWeighted(image, 0.6, heatmap, 0.4, 0)\n\n    # Display original and Grad-CAM overlay side-by-side\n    axs[i, 0].imshow(original_image)\n    axs[i, 0].set_title(f'Original Image {i+1}')\n    axs[i, 0].axis('off')\n\n    axs[i, 1].imshow(overlay)\n    axs[i, 1].set_title(f'Attention Map {i+1}')\n    axs[i, 1].axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, classification_report\npred_Y = retina_model.predict(test_X, batch_size = 32, verbose = True)\npred_Y_cat = np.argmax(pred_Y, -1)\ntest_Y_cat = np.argmax(test_Y, -1)\nprint('Accuracy on Test Data: %2.2f%%' % (accuracy_score(test_Y_cat, pred_Y_cat)))\nprint(classification_report(test_Y_cat, pred_Y_cat))","metadata":{"_uuid":"b421b6183b1919a7414482f0b1ac611079e45174","_cell_guid":"d0edaf00-4b7c-4f65-af0b-e5a03b9b8428","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nsns.heatmap(confusion_matrix(test_Y_cat, pred_Y_cat), \n            annot=True, fmt=\"d\", cbar = False, cmap = plt.cm.Blues, vmax = test_X.shape[0]//16)","metadata":{"_uuid":"10162e055ca7cd52878a289bab377231787ab732","_cell_guid":"15189df2-3fed-495e-9661-97bb2b712dfd","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom sklearn.metrics import roc_curve, roc_auc_score\nsick_vec = test_Y_cat>0\nsick_score = np.sum(pred_Y[:,1:],1)\nfpr, tpr, _ = roc_curve(sick_vec, sick_score)\nfig, ax1 = plt.subplots(1,1, figsize = (6, 6), dpi = 150)\nax1.plot(fpr, tpr, 'b.-', label = 'Model Prediction (AUC: %2.2f)' % roc_auc_score(sick_vec, sick_score))\nax1.plot(fpr, fpr, 'g-', label = 'Random Guessing')\nax1.legend()\nax1.set_xlabel('False Positive Rate')\nax1.set_ylabel('True Positive Rate');","metadata":{"_uuid":"2b2aaee6c83043f721b0c9ed5bc4229eb7165200","_cell_guid":"829475ab-7db2-4421-b9ad-1a51971fd459","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, m_axs = plt.subplots(2, 4, figsize = (32, 20))\nfor (idx, c_ax) in enumerate(m_axs.flatten()):\n    c_ax.imshow(np.clip(test_X[idx]*127+127,0 , 255).astype(np.uint8), cmap = 'bone')\n    c_ax.set_title('Actual Severity: {}\\n{}'.format(test_Y_cat[idx], \n                                                           '\\n'.join(['Predicted %02d (%04.1f%%): %s' % (k, 100*v, '*'*int(10*v)) for k, v in sorted(enumerate(pred_Y[idx]), key = lambda x: -1*x[1])])), loc='left')\n    c_ax.axis('off')\nfig.savefig('trained_img_predictions.png', dpi = 300)","metadata":{"_uuid":"ba87d0e7c3a77181487b99ca64d13de2aa8a21ee","_cell_guid":"c34f049f-b032-45bf-9d5e-a756ecc46a82","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import cv2\nimport numpy as np\nimport torch\nimport torch.nn.functional as F\nfrom torchvision import models, transforms\nimport matplotlib.pyplot as plt\n\n# List of image paths\nimage_paths = [\n    '/kaggle/working/data/train_11/11779_right.jpeg', '/kaggle/working/data/train_11/1177_left.jpeg',\n    '/kaggle/working/data/train_11/1177_right.jpeg', '/kaggle/working/data/train_11/11780_left.jpeg',\n    '/kaggle/working/data/train_11/11780_right.jpeg', '/kaggle/working/data/train_11/11781_left.jpeg',\n    '/kaggle/working/data/train_11/11786_left.jpeg',\n    '/kaggle/working/data/train_11/11786_right.jpeg', '/kaggle/working/data/train_11/11789_left.jpeg',\n    '/kaggle/working/data/train_11/11789_right.jpeg', \n    '/kaggle/working/data/train_11/1178_right.jpeg'\n]\n\n# Step 2: Preprocess the Image\npreprocess = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n# Step 3: Load Pre-trained Model and Set to Evaluation Mode\nmodel = models.resnet50(pretrained=True)\nmodel.eval()\n\n# Step 4: Define Grad-CAM Function\nclass GradCAM:\n    def __init__(self, model, target_layer):\n        self.model = model\n        self.target_layer = target_layer\n        self.gradients = None\n        self.activations = None\n        self.hook_layers()\n\n    def hook_layers(self):\n        def forward_hook(module, input, output):\n            self.activations = output.detach()\n\n        def backward_hook(module, grad_in, grad_out):\n            self.gradients = grad_out[0].detach()\n\n        target_layer = dict(self.model.named_modules())[self.target_layer]\n        target_layer.register_forward_hook(forward_hook)\n        target_layer.register_backward_hook(backward_hook)\n\n    def generate_heatmap(self, input_tensor, class_index=None):\n        # Forward pass\n        output = self.model(input_tensor)\n\n        # Convert output to Tensor if it's not already\n        if isinstance(output, torch.Tensor):\n            output = output\n        else:\n            output = output.data  # Handle old versions of PyTorch\n\n        if class_index is None:\n            # Ensure output is a tensor and use max(1) to get the class index\n            class_index = output.max(1)[1].item()\n\n        score = output[:, class_index]\n\n        # Backward pass\n        self.model.zero_grad()\n        score.backward()\n\n        # Calculate Grad-CAM\n        weights = self.gradients.mean(dim=2, keepdim=True)  # Fix: Use dim=2 for channel dimension\n        cam = (weights * self.activations).sum(dim=1)[0]  # Summing across channels (dim=1)\n        cam = F.relu(cam)  # Apply ReLU to keep positive values\n        cam = cam / cam.max()  # Normalize\n\n        # Resize to match original image size\n        cam = cv2.resize(cam.cpu().numpy(), (224, 224))\n        return cam\n\n# Initialize GradCAM with the target layer\ngrad_cam = GradCAM(model=model, target_layer='layer4')\n\n# Step 5: Loop through each image and display results\nfig, axs = plt.subplots(len(image_paths), 2, figsize=(10, 5 * len(image_paths)))\n\nfor i, image_path in enumerate(image_paths):\n    # Load and preprocess each image\n    image = cv2.imread(image_path)\n    if image is None:\n        print(f\"Image not found at path: {image_path}\")\n        continue\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    original_image = image.copy()\n    image = cv2.resize(image, (224, 224))\n    input_tensor = preprocess(image).unsqueeze(0)  # Add batch dimension\n\n    # Generate Grad-CAM heatmap\n    heatmap = grad_cam.generate_heatmap(input_tensor)\n    heatmap = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)\n    overlay = cv2.addWeighted(image, 0.6, heatmap, 0.4, 0)\n\n    # Display original and Grad-CAM overlay side-by-side\n    axs[i, 0].imshow(original_image)\n    axs[i, 0].set_title(f'Original Image {i+1}')\n    axs[i, 0].axis('off')\n\n    axs[i, 1].imshow(overlay)\n    axs[i, 1].set_title(f'Attention Map {i+1}')\n    axs[i, 1].axis('off')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}