{"cells":[{"metadata":{},"cell_type":"markdown","source":"
\n NOTE: The following notebook it's a modification of this work Triple Stratified KFold with TFRecords, please check it out. \n
","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null\n\nimport pandas as pd, numpy as np\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.layers import (Dropout,\n Conv2D,\n BatchNormalization,\n Dense,\n GlobalAveragePooling2D,\n Input,\n Activation,\n Lambda,\n multiply)\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_style(\"darkgrid\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Configuration ","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"DEVICE = \"TPU\" #or \"GPU\"\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 42\n\n# NUMBER OF FOLDS. USE 3, 5, OR 15 \nFOLDS = 5\n\n# WHICH IMAGE SIZES TO LOAD EACH FOLD\n# CHOOSE 128, 192, 256, 384, 512, 768 \nIMG_SIZES = [384,384,384,384,384]\n\n# INCLUDE OLD COMP DATA? YES=1 NO=0\nINC2019 = [0,0,0,0,0]\nINC2018 = [1,1,1,1,1]\n\n# BATCH SIZE AND EPOCHS\nBATCH_SIZES = [32]*FOLDS\nEPOCHS = [12]*FOLDS\n\n# WHICH EFFICIENTNET B? TO USE\nEFF_NETS = [6,6,6,6,6]\n\n# WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\nWGTS = [1/FOLDS]*FOLDS\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 11","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n print(\"connecting to TPU...\")\n try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n print('Running on TPU ', tpu.master())\n except ValueError:\n print(\"Could not connect to TPU\")\n tpu = None\n\n if tpu:\n try:\n print(\"initializing TPU ...\")\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\n print(\"TPU initialized\")\n except _:\n print(\"failed to initialize TPU\")\n else:\n DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n print(\"Using default strategy for CPU and single GPU\")\n strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n \n\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocess ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS\nfor i,k in enumerate(IMG_SIZES):\n GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))\n GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))\nfiles_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Augmentation ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n # returns 3x3 transformmatrix which transforms indicies\n \n # CONVERT DEGREES TO RADIANS\n rotation = math.pi * rotation / 180.\n shear = math.pi * shear / 180.\n\n def get_3x3_mat(lst):\n return tf.reshape(tf.concat([lst],axis=0), [3,3])\n \n # ROTATION MATRIX\n c1 = tf.math.cos(rotation)\n s1 = tf.math.sin(rotation)\n one = tf.constant([1],dtype='float32')\n zero = tf.constant([0],dtype='float32')\n \n rotation_matrix = get_3x3_mat([c1, s1, zero, \n -s1, c1, zero, \n zero, zero, one]) \n # SHEAR MATRIX\n c2 = tf.math.cos(shear)\n s2 = tf.math.sin(shear) \n \n shear_matrix = get_3x3_mat([one, s2, zero, \n zero, c2, zero, \n zero, zero, one]) \n # ZOOM MATRIX\n zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero, \n zero, one/width_zoom, zero, \n zero, zero, one]) \n # SHIFT MATRIX\n shift_matrix = get_3x3_mat([one, zero, height_shift, \n zero, one, width_shift, \n zero, zero, one])\n \n return K.dot(K.dot(rotation_matrix, shear_matrix), \n K.dot(zoom_matrix, shift_matrix))\n\n\ndef transform(image, DIM=256): \n # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n # output - image randomly rotated, sheared, zoomed, and shifted\n XDIM = DIM%2 #fix for size 331\n \n rot = ROT_ * tf.random.normal([1], dtype='float32')\n shr = SHR_ * tf.random.normal([1], dtype='float32') \n h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n # GET TRANSFORMATION MATRIX\n m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n # LIST DESTINATION PIXEL INDICES\n x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n z = tf.ones([DIM*DIM], dtype='int32')\n idx = tf.stack( [x,y,z] )\n \n # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n idx2 = K.cast(idx2, dtype='int32')\n idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n \n # FIND ORIGIN PIXEL VALUES \n idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n d = tf.gather_nd(image, tf.transpose(idx3))\n \n return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_labeled_tfrecord(example):\n tfrec_format = {\n 'image' : tf.io.FixedLenFeature([], tf.string),\n 'image_name' : tf.io.FixedLenFeature([], tf.string),\n 'patient_id' : tf.io.FixedLenFeature([], tf.int64),\n 'sex' : tf.io.FixedLenFeature([], tf.int64),\n 'age_approx' : tf.io.FixedLenFeature([], tf.int64),\n 'anatom_site_general_challenge': tf.io.FixedLenFeature([], tf.int64),\n 'diagnosis' : tf.io.FixedLenFeature([], tf.int64),\n 'target' : tf.io.FixedLenFeature([], tf.int64)\n } \n example = tf.io.parse_single_example(example, tfrec_format)\n return example['image'], example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n tfrec_format = {\n 'image' : tf.io.FixedLenFeature([], tf.string),\n 'image_name' : tf.io.FixedLenFeature([], tf.string),\n }\n example = tf.io.parse_single_example(example, tfrec_format)\n return example['image'], example['image_name'] if return_image_name else 0\n\n \ndef prepare_image(img, augment=True, dim=256): \n img = tf.image.decode_jpeg(img, channels=3)\n img = tf.cast(img, tf.float32) / 255.0\n \n if augment:\n img = transform(img,DIM=dim)\n img = tf.image.random_flip_left_right(img)\n #img = tf.image.random_hue(img, 0.01)\n img = tf.image.random_saturation(img, 0.7, 1.3)\n img = tf.image.random_contrast(img, 0.8, 1.2)\n img = tf.image.random_brightness(img, 0.1)\n \n img = tf.reshape(img, [dim,dim, 3])\n \n return img\n\ndef count_data_items(filenames):\n n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n for filename in filenames]\n return np.sum(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n labeled=True, return_image_names=True, batch_size=16, dim=256):\n \n ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n ds = ds.cache()\n \n if repeat:\n ds = ds.repeat()\n \n if shuffle: \n ds = ds.shuffle(1024*8)\n opt = tf.data.Options()\n opt.experimental_deterministic = False\n ds = ds.with_options(opt)\n \n if labeled: \n ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n else:\n ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n num_parallel_calls=AUTO) \n \n ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, dim=dim), \n imgname_or_label), \n num_parallel_calls=AUTO)\n \n ds = ds.batch(batch_size * REPLICAS)\n ds = ds.prefetch(AUTO)\n return ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building the Model with Attention ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]\n\ndef build_model(dim = 128, ef = 0):\n ### Base Model ###\n # Input\n inp = Input(shape = (dim, dim, 3))\n # Base EfficientNet pretrained model\n base = EFNS[ef](\n input_shape = (dim, dim, 3),\n weights = \"imagenet\",\n include_top = False,\n drop_connect_rate = 0.3\n )\n # variables for the attention mechanism and later\n pt_depth = base.get_output_shape_at(0)[-1]\n pt_features = base(inp)\n bn_features = BatchNormalization()(pt_features)\n\n ### Attention Mechanism ###\n attn_layer = Conv2D(64, kernel_size = (1, 1), padding = \"same\", activation = \"relu\")(Dropout(0.5)(bn_features))\n attn_layer = Conv2D(16, kernel_size = (1, 1), padding = \"same\", activation = \"relu\")(attn_layer)\n attn_layer = Conv2D(8, kernel_size = (1, 1), padding = \"same\", activation = \"relu\")(attn_layer)\n attn_layer = Conv2D(1, kernel_size = (1, 1), padding = \"valid\", activation = \"sigmoid\")(attn_layer)\n\n # Fan it out to all of the channels\n up_c2_w = np.ones((1, 1, 1, pt_depth))\n up_c2 = Conv2D(\n pt_depth, kernel_size = (1, 1),\n padding = \"same\",\n activation = \"linear\",\n use_bias = False,\n weights = [up_c2_w]\n )\n up_c2.trainable = False\n attn_layer = up_c2(attn_layer)\n\n mask_features = multiply([attn_layer, bn_features])\n gap_features = GlobalAveragePooling2D()(mask_features)\n gap_mask = GlobalAveragePooling2D()(attn_layer)\n\n # To account for missing values from the attention model\n gap = Lambda(lambda x: x[0] / x[1], name = \"RescaleGAP\")([gap_features, gap_mask])\n gap_dr = Dropout(0.25)(gap)\n dr_steps = Dropout(0.25)(Dense(128, activation = \"relu\")(gap_dr))\n\n ### Rebuild top ###\n x = Dense(1, activation = \"sigmoid\")(dr_steps)\n\n ### Compile ###\n model = Model(inputs = inp, outputs = x)\n # Optimizer\n opt = tf.keras.optimizers.Adam(learning_rate = 0.001)\n # Loss\n loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.05) \n model.compile(optimizer = opt, loss = loss, metrics = [\"AUC\"])\n return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Schedule ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n lr_start = 0.000005\n lr_max = 0.00000125 * REPLICAS * batch_size\n lr_min = 0.000001\n lr_ramp_ep = 5\n lr_sus_ep = 0\n lr_decay = 0.8\n \n def lrfn(epoch):\n if epoch < lr_ramp_ep:\n lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n \n elif epoch < lr_ramp_ep + lr_sus_ep:\n lr = lr_max\n \n else:\n lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n \n return lr\n\n lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n return lr_callback","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 0\nDISPLAY_PLOT = True\n\nskf = KFold(n_splits = FOLDS,shuffle = True,random_state = SEED)\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(15))):\n \n # DISPLAY FOLD INFO\n if DEVICE=='TPU':\n if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n print('#'*25); print('#### FOLD',fold+1)\n print('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n (IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))\n \n # CREATE TRAIN AND VALIDATION SUBSETS\n files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])\n if INC2019[fold]:\n files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])\n print('#### Using 2019 external data')\n if INC2018[fold]:\n files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])\n print('#### Using 2018+2017 external data')\n np.random.shuffle(files_train); print('#'*25)\n files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])\n files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))\n \n # BUILD MODEL\n K.clear_session()\n with strategy.scope():\n model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])\n \n # SAVE BEST MODEL EACH FOLD\n sv = tf.keras.callbacks.ModelCheckpoint(\n 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n save_weights_only=True, mode='min', save_freq='epoch')\n \n # TRAIN\n print('Training...')\n history = model.fit(\n get_dataset(\n files_train,\n augment = True, \n shuffle = True, \n repeat = True,\n dim = IMG_SIZES[fold],\n batch_size = BATCH_SIZES[fold]\n ), \n epochs = EPOCHS[fold],\n callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])], \n steps_per_epoch = count_data_items(files_train) / BATCH_SIZES[fold]//REPLICAS,\n validation_data = get_dataset(\n files_valid,augment = False,\n shuffle = False,\n repeat = False,\n dim = IMG_SIZES[fold]\n ), # class_weight = {0:1,1:2},\n verbose = VERBOSE\n )\n \n print('Loading best model...')\n model.load_weights('fold-%i.h5'%fold)\n \n # PREDICT OOF USING TTA\n print('Predicting OOF with TTA...')\n ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,\n repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS\n pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,] \n oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) ) \n #oof_pred.append(model.predict(get_dataset(files_valid,dim=IMG_SIZES[fold]),verbose=1))\n \n # GET OOF TARGETS AND NAMES\n ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n labeled=True, return_image_names=True)\n oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n labeled=False, return_image_names=True)\n oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n \n # PREDICT TEST USING TTA\n print('Predicting Test with TTA...')\n ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,\n repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS\n pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,] \n preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) * WGTS[fold]\n \n # REPORT RESULTS\n auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n oof_val.append(np.max( history.history['val_auc'] ))\n print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n \n # PLOT TRAINING\n if DISPLAY_PLOT:\n plt.figure(figsize=(15,5))\n plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n plt.legend(loc=2)\n plt2 = plt.gca().twinx()\n plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n ydist = plt.ylim()[1] - plt.ylim()[0]\n plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n plt.ylabel('Loss',size=14)\n plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'%\n (fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)\n plt.legend(loc=3)\n plt.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Calculate OOF AUC ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# COMPUTE OVERALL OOF AUC\noof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);\nnames = np.concatenate(oof_names); folds = np.concatenate(oof_folds)\nauc = roc_auc_score(true,oof)\nprint('Overall OOF AUC with TTA = %.3f'%auc)\n\n# SAVE OOF TO DISK\ndf_oof = pd.DataFrame(dict(\n image_name = names, target=true, pred = oof, fold=folds))\ndf_oof.to_csv('oof.csv',index=False)\ndf_oof.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Post Process","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],\n labeled=False, return_image_names=True)\n\nimage_names = np.array([img_name.numpy().decode(\"utf-8\") \n for img, img_name in iter(ds.unbatch())])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submit to Kaggle ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(dict(image_name=image_names, target=preds[:,0]))\nsubmission = submission.sort_values('image_name') \nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission.target,bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# References\n\n- [Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords)\n- [Melanoma Classification with Attention!](https://www.kaggle.com/ibtesama/melanoma-classification-with-attention)\n- [APTOS:Keras Efficientnet with Attention Baseline](https://www.kaggle.com/ratan123/aptos-keras-efficientnet-with-attention-baseline)\n- [Attention on Pretrained-VGG16 for Bone Age](https://www.kaggle.com/kmader/attention-on-pretrained-vgg16-for-bone-age)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}