{"cells":[{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"\n!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":true},"cell_type":"code","source":"import os\nimport cv2\nimport math\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.metrics import confusion_matrix,accuracy_score,roc_auc_score,precision_score,recall_score,ConfusionMatrixDisplay\nfrom sklearn.model_selection import train_test_split,KFold,StratifiedKFold\nfrom transformers import get_cosine_schedule_with_warmup\nfrom albumentations import *\nfrom kaggle_datasets import KaggleDatasets\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Model,Sequential\nfrom tensorflow.keras import optimizers\nimport efficientnet.tfkeras as efn\nfrom albumentations import *\n\n\n\nimport warnings  \nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n    \nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"SEED = 42\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync\nSIZE = [512,512]\nIMAGE_SIZE=[512,512]\nLR = 0.0008\nWEIGHT_DECAY = 0\nEPOCHS = 40\nWARMUP = 15\nTTA = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def seed_everything(SEED):\n    np.random.seed(SEED)\n    tf.random.set_seed(SEED)\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COMPETITION_NAME = \"ranzcr-clip-catheter-line-classification\"\n#strategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 8\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(COMPETITION_NAME)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"load_dir = f\"/kaggle/input/{COMPETITION_NAME}/\"\ndf = pd.read_csv(load_dir + 'train.csv')\n\n# paths = load_dir + \"train/\" + df['StudyInstanceUID'] + '.jpg'\npaths = GCS_DS_PATH + \"/train/\" + df['StudyInstanceUID'] + '.jpg'\n\nsub_df = pd.read_csv(load_dir + 'sample_submission.csv')\n\n# test_paths = load_dir + \"test/\" + sub_df['StudyInstanceUID'] + '.jpg'\ntest_paths = GCS_DS_PATH + \"/test/\" + sub_df['StudyInstanceUID'] + '.jpg'\n\n# Get the multi-labels\nlabel_cols = sub_df.columns[1:]\nlabels = df[label_cols].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = list(df.columns[1:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train,valid = train_test_split(df,test_size = 0.10,random_state = SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"transform = {\n    'train' :Compose([\n        Resize(SIZE[0],SIZE[1],always_apply=True),\n        HorizontalFlip(p=0.5),\n        VerticalFlip(p=0.5),\n        RandomRotate90(p=0.5),\n        Rotate(limit=25.0,p=0.8)]) }\n\ndef preprocess(df,test=False):\n    paths = df.StudyInstanceUID.apply(lambda x: GCS_DS_PATH + '/train/' + x + '.jpg').values\n    labels = df.loc[:,'ETT - Abnormal':'Swan Ganz Catheter Present'].values\n    if test==False:\n        return paths,labels\n    else:\n        return paths\n    \ndef decode_image(filename, label=None, image_size=(SIZE[0], SIZE[1])):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3) \n    image = tf.image.resize(image, image_size)\n    image = tf.cast(image, tf.float32)\n    image = tf.image.per_image_standardization(image)\n    if label is None:\n        return image\n    else:\n        return image, label\n    \ndef data_augment(image, label=None, seed=SEED):\n    image = tf.image.rot90(image,k=np.random.randint(4))\n    image = tf.image.random_flip_left_right(image, seed=seed)\n    image = tf.image.random_flip_up_down(image, seed=seed)\n    if label is None:\n        return image\n    else:\n        return image, label\n    \ndef albu(image):\n    transforms = transform['train']\n    image = transforms(image=image.numpy())['image']\n    image = tf.cast(image, tf.float32)\n    return image\n    \ndef albu_fn(image,label=None):\n    [image,] = tf.py_function(albu, [image], [tf.float32])\n    if label is None:\n        return image\n    else:\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\n\ndef transform(image, label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]), label\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train_dataset = (tf.data.Dataset\n    .from_tensor_slices(preprocess(train))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .map(transform, num_parallel_calls=AUTO)\n    .shuffle(SEED)\n    .batch(BATCH_SIZE,drop_remainder=True)\n    .repeat()\n    .prefetch(AUTO))\n\nvalid_dataset = (tf.data.Dataset\n    .from_tensor_slices(preprocess(valid))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"def plot_transform(image_id,num_images=7):\n    plt.figure(figsize=(30,10))\n    path,_ = preprocess(train.iloc[image_id:image_id+1])\n    for i in range(1,num_images+1):\n        plt.subplot(1,num_images+1,i)\n        plt.axis('off')\n        image = decode_image(filename=path[0])\n        image = data_augment(image=image)\n        plt.imshow(image)\n\nplot_transform(9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def categorical_focal_loss(gamma=2., alpha=.25):\n    def categorical_focal_loss_fixed(y_true, y_pred):\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n        epsilon = K.epsilon()\n        y_pred = K.clip(y_pred, epsilon, 1. - epsilon)\n        cross_entropy = -y_true * K.log(y_pred)\n        loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy\n        return K.sum(loss, axis=1)\n    return categorical_focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential, load_model\nfrom keras.layers import (Activation, Dropout, Flatten, Dense, GlobalMaxPooling2D,\n                          BatchNormalization, Input, Conv2D, GlobalAveragePooling2D,concatenate,Concatenate,multiply, LocallyConnected2D, Lambda)\nfrom keras.callbacks import ModelCheckpoint\nfrom keras import metrics\nfrom keras.optimizers import Adam \nimport keras\nfrom keras.models import Model\nfrom keras.activations import hard_sigmoid\nCFG = dict(\n    inp_size          = 512,\n    read_size         = 512, \n    crop_size         = 512,\n    net_size          = 512)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im_size=512\nn_labels = labels.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    in_lay = Input(shape=(im_size, im_size,3))\n    base_model = tf.keras.applications.InceptionV3(weights='imagenet',\n        input_shape=(im_size, im_size,3),\n        include_top=False\n                       )\n    #base_model.load_weights(\"../input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5\")\n    pt_depth = base_model.get_output_shape_at(0)[-1]\n    pt_features = base_model(in_lay)\n    bn_features = BatchNormalization()(pt_features)\n    \n    # ici nous faisons un mécanisme d'attention pour activer et désactiver les pixels dans le GAP\n    # lidee est baser sur cette explication \n    #1-http://akosiorek.github.io/ml/2017/10/14/visual-attention.html\n    #2-https://machinelearningmastery.com/how-does-attention-work-in-encoder-decoder-recurrent-neural-networks/\n    \n    \n    attn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\n    attn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n    attn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n    attn_layer = Conv2D(1, \n                        kernel_size = (1,1), \n                        padding = 'valid', \n                        activation = 'sigmoid')(attn_layer)\n    # diffusez sur toutes les chaînes\n    # kernel_size  détermine les dimensions du noyau. Les dimensions courantes comprennent 1×1, 3×3, 5×5 et 7×7, qui peuvent être passées en (1, 1), (3, 3), (5, 5) ou (7, 7) tuples.\n    # Il s'agit d'un nombre entier ou d'un tuple/liste de 2 nombres entiers, spécifiant la hauteur et la largeur de la fenêtre de convolution 2D.\n    #  Ce paramètre doit être un nombre entier impair\n    # pour plus de details sur cette partie (mask et use_bias ... ) il ya  une bonne explication sur geekforgeeks\n    #https://www.geeksforgeeks.org/keras-conv2d-class/\n    \n    up_c2_w = np.ones((1, 1, 1, pt_depth))\n    up_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', \n                   activation = 'linear', use_bias = False, weights = [up_c2_w])\n    up_c2.trainable = False\n    attn_layer = up_c2(attn_layer)\n\n    mask_features = multiply([attn_layer, bn_features])\n    gap_features = GlobalAveragePooling2D()(mask_features)\n    gap_mask = GlobalAveragePooling2D()(attn_layer)\n    \n    # pour tenir compte des valeurs manquantes du modèle d'attention\n    # pour bien comprendre resaclegap il ya un bon exemple ici qui explique tellemnt bien cette partie \n    # https://codefellows.github.io/sea-python-401d5/lectures/rescaling_data.html\n    \n    gap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\n    gap_dr = Dropout(0.5)(gap)\n    dr_steps = Dropout(0.5)(Dense(64, activation = 'relu')(gap_dr))\n    out_layer = Dense(n_labels , activation = 'sigmoid')(dr_steps)\n    model = Model(inputs = [in_lay], outputs = [out_layer])  \n    model.compile(\n        optimizer=tf.keras.optimizers.Adam(),\n        loss='binary_crossentropy',\n        metrics=[tf.keras.metrics.AUC(multi_label=True)])\n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"EPOCHS_ = 20\nLR_START = 0.0001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.000005\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 3\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = np.random.random_sample() * LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nrng = [i for i in range(EPOCHS_)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = tf.keras.callbacks.ModelCheckpoint(\n    'model.h5', save_best_only=True, monitor='val_auc', mode='max')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"#STEPS_PER_EPOCH = train.shape[0] // BATCH_SIZE\ndef train():\n    history = model.fit(\n        train_dataset, \n        epochs=EPOCHS_, \n        callbacks=[checkpoint,lr_callback],\n        steps_per_epoch=200,  \n        validation_data=valid_dataset)\n    \n    string = 'Train acc:{:.4f} Train loss:{:.4f},Val acc:{:.4f} Val loss:{:.4f}'.format( \\\n        model.history.history['categorical_accuracy'][-1],model.history.history['loss'][-1],\\\n        model.history.history['val_categorical_accuracy'][-1],model.history.history['val_loss'][-1])\n    \n    return string","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def display_training_curves(training, validation, title, subplot):\n    \"\"\"\n    Source: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n    \"\"\"\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(15,15), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"display_training_curves(\n    model.history.history['loss'], \n    model.history.history['val_loss'], \n    'loss', 211)\ndisplay_training_curves(\n    model.history.history['categorical_accuracy'], \n    model.history.history['val_categorical_accuracy'], \n    'accuracy', 212)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}