{"cells":[{"metadata":{},"cell_type":"markdown","source":"[pour la visualisation et la compréhension de notre dataset j'ai met en public ce travail sur kaggle ](https://www.kaggle.com/tikoboss/skin-cancer-siim-melanoma)\n"},{"metadata":{},"cell_type":"markdown","source":"[pour la création de tfrecords ](https://www.kaggle.com/cdeotte/how-to-create-tfrecords)\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"import math, re, os\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nimport numpy as np\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix, accuracy_score\nimport datetime\nimport tqdm\nimport json\nfrom collections import Counter\nimport gc\ngc.enable()\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configurations"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Configuration\nIMAGE_SIZE = [128,128]\nBATCH_SIZE = 16* strategy.num_replicas_in_sync\n\nEPOCHS = 15","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data paths"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('oc-d-512512') # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"\nCLASSES=[0,1,2,3,4,5,6,7]\nIMAGE_SIZE = [512, 512] # at this size, a GPU will run out of memory. Use the TPU\nEPOCHS = 50\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nfrom sklearn.model_selection import train_test_split\nGCS_PATH    = KaggleDatasets().get_gcs_path('oc-d-512512')\n\nTRAINING_FILENAMES = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/train*.tfrec')))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##### split data-set 80% training 20% validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAINING_FILENAMES,VALIDATION_FILENAMES = train_test_split(TRAINING_FILENAMES,test_size = 0.20,random_state =42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Classes"},{"metadata":{"trusted":true},"cell_type":"code","source":"CLASSES = [0,1]                                                                                                                                               # 100 - 102","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset Functions\nbaser sur ce   [kernel][1]\n\n[1]: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu"},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"filename\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['filename']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO) # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    return dataset\n\ndef get_training_dataset():\n    \n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.repeat() # Since we use custom training loop, we don't need to use repeat() here.\n    dataset = dataset.shuffle(20000)\n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=False)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    \n    return dataset  \n\ndef get_validation_dataset():\n    \n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=True)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = int(count_data_items(TRAINING_FILENAMES))\nNUM_VALIDATION_IMAGES = int(count_data_items(VALIDATION_FILENAMES))\n#NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nprint('Dataset: {} training images, {} validation images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Oversample\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get labels and their countings\n\ndef get_training_dataset_raw():\n\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True, ordered=False)\n    return dataset\n\n\nraw_training_dataset = get_training_dataset_raw()\n\nlabel_counter = Counter()\nfor images, labels in raw_training_dataset:\n    label_counter.update([labels.numpy()])\n\ndel raw_training_dataset    \n    \nlabel_counting_sorted = label_counter.most_common()\n\nNUM_TRAINING_IMAGES = sum([x[1] for x in label_counting_sorted])\nprint(\"number of examples in the original training dataset: {}\".format(NUM_TRAINING_IMAGES))\n\nprint(\"labels in the original training dataset, sorted by occurrence\")\nlabel_counting_sorted","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## nombre de repetition de chaque classe\npreparer les fonction qui nous aide  pour faire(le oversampling)\ntrouver le nombre de repetitions de chaque classe dans le dataset afin d'équilibrer les classes "},{"metadata":{"trusted":true},"cell_type":"code","source":"#  Nous voulons que chaque classe se produise au moins (environ) fois `TARGET_MIN_COUNTING`\nTARGET_MIN_COUNTING = 100 # chaque classe va se produire 100 fois \n\ndef get_num_of_repetition_for_class(class_id):\n    \n    counting = label_counter[class_id]\n    if counting >= TARGET_MIN_COUNTING:\n        return 1.0\n    \n    num_to_repeat = TARGET_MIN_COUNTING / counting\n    \n    return num_to_repeat\n\nnumbers_of_repetition_for_classes = {class_id: get_num_of_repetition_for_class(class_id) for class_id in range(8)}\n\nprint(\"number of repetitions for each class (if > 1)\")\n{k: v for k, v in sorted(numbers_of_repetition_for_classes.items(), key=lambda item: item[1], reverse=True) if v > 1}\n\nkeys_tensor = tf.constant([k for k in numbers_of_repetition_for_classes])\nvals_tensor = tf.constant([numbers_of_repetition_for_classes[k] for k in numbers_of_repetition_for_classes])\ntable = tf.lookup.StaticHashTable(tf.lookup.KeyValueTensorInitializer(keys_tensor, vals_tensor), -1)\n\ndef get_num_of_repetition_for_example(training_example):\n    \n    _, label = training_example\n    \n    num_to_repeat = table.lookup(label)\n    num_to_repeat_integral = tf.cast(int(num_to_repeat), tf.float32)\n    residue = num_to_repeat - num_to_repeat_integral\n    \n    num_to_repeat = num_to_repeat_integral + tf.cast(tf.random.uniform(shape=()) <= residue, tf.float32)\n    \n    return tf.cast(num_to_repeat, tf.int64)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"pour le oversampling il ya des techniques specifiques a cause q'uil sagit des fichiers tfrecords donc on peut pas utiliser les  techniques classiques pour le oversampling\ntelle que smote ....\n\ndonc on peut utiliser les poids de cette facon :\n\n    from sklearn.utils import class_weight\n    train = pd.read_csv('../input/siim-isic-melanoma-classification/train.csv')\n    class_weights = class_weight.compute_class_weight('balanced',\n                                                     np.unique(train.target),\n                                                     train.target)\n    class_weights = dict(enumerate(class_weights))\n    \nremarque : dans notre class_weights n'ajoute pas vraiment ce que nous attends d'elle  donc je l'ai pas utiliser"},{"metadata":{},"cell_type":"markdown","source":"aussi on peut utiliser ce code pour le oversampling \n\n    import pandas as pd\n    from collections import Counter\n\n    def get_class_weights(y):\n        counter = Counter(y)\n        majority = max(counter.values())\n        return  {cls: round(float(majority)/float(count), 2) for cls, count in counter.items()}\n\n    train = pd.read('train.csv')\n    class_weights = get_class_weights(train.is_attributed.values)\n    print(class_weights)"},{"metadata":{},"cell_type":"markdown","source":"## augmentation "},{"metadata":{},"cell_type":"markdown","source":"pour l'augmentation il ya plusieurs techniques a utiliser donc moi jai choiser cette selon cette explication dans ce kernel \n\n[l'augmentation est baser sur ce kernel](https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96)\n\n\npour mes prochain travaux on peut utiliser cette technique :\n\n[source:](https://www.kaggle.com/atamazian/fc-ensemble-external-data-effnet-densenet)\n\n    def random_blockout(img, sl=0.1, sh=0.2, rl=0.4):\n        p=random.random()\n        if p>=0.25:\n            w, h, c = IMAGE_SIZE[0], IMAGE_SIZE[1], 3\n            origin_area = tf.cast(h*w, tf.float32)\n\n        e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)), tf.int32)\n        e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)), tf.int32)\n\n        e_height_h = tf.minimum(e_size_h, h)\n        e_width_h = tf.minimum(e_size_h, w)\n\n        erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)\n        erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)\n\n        erase_area = tf.zeros(shape=[erase_height, erase_width, c])\n        erase_area = tf.cast(erase_area, tf.uint8)\n\n        pad_h = h - erase_height\n        pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)\n        pad_bottom = pad_h - pad_top\n\n        pad_w = w - erase_width\n        pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)\n        pad_right = pad_w - pad_left\n\n        erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)\n        erase_mask = tf.squeeze(erase_mask, axis=0)\n        erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))\n\n        return tf.cast(erased_img, img.dtype)\n    else:\n        return tf.cast(img, img.dtype)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\n\ndef transform(image, label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3]), label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"[pour voir le test sur l'augmentation j'ai créer un kernel sur kaggle ou je montre le fonctionement de notre augmentation avec les images ](https://www.kaggle.com/tikoboss/test-of-aug-by-tko-okt?scriptVersionId=42269313)"},{"metadata":{},"cell_type":"markdown","source":"## oversampled training dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_dataset_with_oversample(repeat_dataset=True, oversample=False, augumentation=False):\n\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n\n    if oversample:\n        dataset = dataset.flat_map(lambda image, label: tf.data.Dataset.from_tensors((image, label)).repeat(get_num_of_repetition_for_example((image, label))))\n\n    if augumentation:\n        dataset = dataset.map(transform, num_parallel_calls=AUTO)\n    \n    if repeat_dataset:\n        dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    \n    dataset = dataset.shuffle(20000)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## test for oversampled dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"oversampled_training_dataset = get_training_dataset_with_oversample(repeat_dataset=False, oversample=True, augumentation=False)\n\nlabel_counter_2 = Counter()\nfor images, labels in oversampled_training_dataset:\n    label_counter_2.update(labels.numpy())\n\ndel oversampled_training_dataset\n\nlabel_counting_sorted_2 = label_counter_2.most_common()\n\nNUM_TRAINING_IMAGES_OVERSAMPLED = sum([x[1] for x in label_counting_sorted_2])\nprint(\"number of examples in the oversampled training dataset: {}\".format(NUM_TRAINING_IMAGES_OVERSAMPLED))\n\nprint(\"labels in the oversampled training dataset, sorted by occurrence\")\nlabel_counting_sorted_2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"dans d'autres problemes on peut utiliser cette fonction f1_loss pour minimiser la perte de f1_score ,qui fait que f1_score va augmenter\n\n    def f1_loss(y_true, y_pred):\n\n        tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n        tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n        fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n        fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n        p = tp / (tp + fp + K.epsilon())\n        r = tp / (tp + fn + K.epsilon())\n\n        f1 = 2*p*r / (p+r+K.epsilon())\n        f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n        return 1 - K.mean(f1)"},{"metadata":{},"cell_type":"markdown","source":"pour calculer f1_score avec macro aussi je suggéré cette façon d'utilisation de f1_score\nmetrics = [tfa.metrics.f_scores.F1Score(num_classes=2,average=\"macro\")] \n\n\n    def fbeta_score_macro(y_true, y_pred, beta=1, threshold=0.1):\n\n        y_true = K.cast(y_true, 'float')\n        y_pred = K.cast(K.greater(K.cast(y_pred, 'float'), threshold), 'float')\n\n        tp = K.sum(y_true * y_pred, axis=0)\n        fp = K.sum((1 - y_true) * y_pred, axis=0)\n        fn = K.sum(y_true * (1 - y_pred), axis=0)\n\n        p = tp / (tp + fp + K.epsilon())\n        r = tp / (tp + fn + K.epsilon())\n\n        f1 = (1 + beta ** 2) * p * r / ((beta ** 2) * p + r + K.epsilon())\n        f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n        return K.mean(f1)"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nimport tensorflow as tf\nfrom keras.losses import binary_crossentropy, categorical_crossentropy\nimport keras.backend as K\nimport numpy as np\nfrom prettytable import PrettyTable\nfrom prettytable import ALL\nfrom sklearn.metrics import f1_score\nfrom matplotlib import pyplot as plt\n\ndef f1(y_true, y_pred):\n    y_pred = K.round(y_pred)\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\n\ndef f1_loss(y_true, y_pred):\n    \n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = 2*p*r / (p+r+K.epsilon())\n    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n    return 1 - K.mean(f1)\n\n\ndef fbeta_score_macro(y_true, y_pred, beta=1, threshold=0.1):\n\n    y_true = K.cast(y_true, 'float')\n    y_pred = K.cast(K.greater(K.cast(y_pred, 'float'), threshold), 'float')\n\n    tp = K.sum(y_true * y_pred, axis=0)\n    fp = K.sum((1 - y_true) * y_pred, axis=0)\n    fn = K.sum(y_true * (1 - y_pred), axis=0)\n\n    p = tp / (tp + fp + K.epsilon())\n    r = tp / (tp + fn + K.epsilon())\n\n    f1 = (1 + beta ** 2) * p * r / ((beta ** 2) * p + r + K.epsilon())\n    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\n    \n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# learning rate"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Learning rate schedule for TPU, GPU and CPU.\n# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\n\n# pour fine-tuning https://flyyufelix.github.io/2016/10/03/fine-tuning-in-keras-part1.html\n\nLR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n\nrng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> #aussi pour learning rate \n\n*  source [get_cosine_schedule_with_warmup](https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_schedule_with_warmup)"},{"metadata":{},"cell_type":"markdown","source":"aussi pour LR on peut utiliser cette fonction que je trouve vraiment efficace \n[source ](https://www.kaggle.com/chankhavu/a-beginner-s-tpu-kernel-single-model-0-97)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import kernel_tensorflow_utils as ktu\nlr_callback = ktu.LRSchedulers.FineTuningLR(\n    \n    lr_start=1e-5, lr_max=5e-5 * strategy.num_replicas_in_sync, lr_min=1e-5,\n    lr_rampup_epochs=5, lr_sustain_epochs=0, lr_exp_decay=0.8, verbose=1)\n\nplt.figure(figsize=(8, 5))\nlr_callback.visualize(steps_per_epoch=NUM_TRAINING_IMAGES, epochs=40)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds = get_validation_dataset()\n\nvalid_images_ds = valid_ds.map(lambda image, label: image)\nvalid_labels_ds = valid_ds.map(lambda image, label: label).unbatch()\n\nvalid_labels = next(iter(valid_labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() # get everything as one batch\n\nvalid_steps = NUM_VALIDATION_IMAGES // BATCH_SIZE\n\nif NUM_VALIDATION_IMAGES % BATCH_SIZE > 0:\n    valid_steps += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train +original dataset\nen utilisant dataset tele quelle est ,pas de aug et pas de oversample"},{"metadata":{},"cell_type":"markdown","source":"### DenseNet201 sans oversample et sans augumentation\nj'ai déjà essayé **densnet121** et **densnet169** donc les meilleurs résultats c'est en utilisant DenseNet201\n,aussi **densenet201** sans oversample et avec aug il n'ya pas vraiment de changement en terme de résultats par contre une perte de 2% en niveau de f1_score et une grande perte de 4% en niveau de accuracy\n\n**remarque** les autres architectures telle que **vgg16** et **vgg19** et **rasnet50**..... ne donne pas de bonne résultats \npar contre j'ai pas essayer **rasnet34** et **rasnet18** ( a base de pytorsh ) \n**rasnext50** j'ai vu un kernel sur ca en utilisant l'ensemble learning mais pas pour le probleme de **skin cancer** donc j'ai pas vraiment essayer ca\n\npour lensemble learning (ensemble de efficienet de b0->b7)  j'ai essayer cette facont mais j'ai eu les memes résultats q'un seul model en + le probleme  cest lors de l'exécution ça prend 3h donc c'est pas le bon choix  \n\npar contre j'ai pas essayer l'ensemble learning de (densenet201,densenet121,densenet169)\n\n### aussi je suggère cette technique dans les prochain travaux\n#### BiLinear EfficientNet Focal Loss+ Label Smoothing\n#### je pense il y'a un seul un articele qui a utiliser cette technique\n[source](https://www.kaggle.com/jimitshah777/bilinear-efficientnet-focal-loss-label-smoothing)"},{"metadata":{},"cell_type":"markdown","source":"pour la perte j'ai essayer cette fonction qui abouti vraiment a de bonne resultas en terme de auc \n\n    def focal_loss(alpha=0.25,gamma=2.0):\n        def focal_crossentropy(y_true, y_pred):\n            bce = K.binary_crossentropy(y_true, y_pred)\n\n            y_pred = K.clip(y_pred, K.epsilon(), 1.- K.epsilon())\n            p_t = (y_true*y_pred) + ((1-y_true)*(1-y_pred))\n\n            alpha_factor = 1\n            modulating_factor = 1\n\n            alpha_factor = y_true*alpha + ((1-alpha)*(1-y_true))\n            modulating_factor = K.pow((1-p_t), gamma)\n\n            # compute the final loss and return\n            return K.mean(alpha_factor*modulating_factor*bce, axis=-1)\n        return focal_crossentropy\n        \n[source focal_loss](https://www.kaggle.com/c/santander-customer-transaction-prediction/discussion/83363)"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install livelossplot\nfrom livelossplot import PlotLossesKeras\ncb=[PlotLossesKeras()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def focal_loss(alpha=0.25,gamma=2.0):\n    def focal_crossentropy(y_true, y_pred):\n        bce = K.binary_crossentropy(y_true, y_pred)\n\n        y_pred = K.clip(y_pred, K.epsilon(), 1.- K.epsilon())\n        p_t = (y_true*y_pred) + ((1-y_true)*(1-y_pred))\n\n        alpha_factor = 1\n        modulating_factor = 1\n\n        alpha_factor = y_true*alpha + ((1-alpha)*(1-y_true))\n        modulating_factor = K.pow((1-p_t), gamma)\n\n        # compute the final loss and return\n        return K.mean(alpha_factor*modulating_factor*bce, axis=-1)\n    return focal_crossentropy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"original_training_dataset = get_training_dataset_with_oversample(repeat_dataset=True, oversample=False, augumentation=False)\n\nimport keras\n\nwith strategy.scope():\n    base_network = efn.EfficientNetB7(input_shape=(512,512,3),weights='noisy-student',include_top=False)\n    network = keras.Sequential()\n    network.add(base_network)\n    network.add(keras.layers.MaxPooling2D())\n    network.add(keras.layers.Conv2D(2810,3,padding='same'))\n    network.add(keras.layers.Dropout(0.25))\n    network.add(keras.layers.BatchNormalization())\n    network.add(keras.layers.ReLU())\n    network.add(keras.layers.GlobalAveragePooling2D())\n\n    network.add(keras.layers.Dense(1024))\n    network.add(keras.layers.Dropout(0.25))\n    network.add(keras.layers.BatchNormalization())\n    network.add(keras.layers.LeakyReLU())\n\n    network.add(keras.layers.Dense(512))\n    network.add(keras.layers.Dropout(0.25))\n    network.add(keras.layers.BatchNormalization())\n    network.add(keras.layers.LeakyReLU())\n\n    network.add(keras.layers.Dense(256))\n    network.add(keras.layers.Dropout(0.25))\n    network.add(keras.layers.BatchNormalization())\n    network.add(keras.layers.LeakyReLU())\n\n\n    network.add(keras.layers.Dense(8,activation='softmax'))\n    network.compile(optimizer=keras.optimizers.Adam(lr=0.00001),loss=keras.losses.SparseCategoricalCrossentropy(),metrics=[keras.metrics.SparseCategoricalAccuracy()])\n    network.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"network.summary()\nfrom keras.utils.vis_utils import plot_model\nplot_model(network, to_file='model_plot.png', show_shapes=True, show_layer_names=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = network.fit(\n    original_training_dataset, \n    steps_per_epoch=NUM_TRAINING_IMAGES // BATCH_SIZE,\n    epochs=12,\n    callbacks=[lr_callback,cb],\n    validation_data=valid_ds\n)\n\nvalid_probs =network.predict(valid_images_ds, steps=valid_steps)\nvalid_preds = np.argmax(valid_probs, axis=-1)\n\ndel network\ngc.collect()\ntf.keras.backend.clear_session()\n\nval_acc = history.history['val_sparse_categorical_accuracy']\n\nscore = f1_score(valid_labels, valid_preds, labels=range(len(CLASSES)), average='macro')\nacc = accuracy_score(valid_labels, valid_preds)\nprecision = precision_score(valid_labels, valid_preds, labels=range(len(CLASSES)), average='macro')\nrecall = recall_score(valid_labels, valid_preds, labels=range(len(CLASSES)), average='macro')\n\nprint(\"results\")\nprint(\"best 10 validation accuracies = {}\".format(sorted(val_acc, reverse=True)[:10]))\nprint('f1 score: {:.6f} | recall: {:.6f} | precision: {:.6f} | acc: {:.6f}'.format(score, recall, precision, acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn.metrics\nprint(sklearn.metrics.classification_report(valid_labels, valid_preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ncm=confusion_matrix(valid_labels, valid_preds, labels=[0,1,2,3,4,5,6,7])\ncm","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}