{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Hi, in this notebook, i try training with swin tranform model\n## Dataset: https://www.kaggle.com/datasets/jpbremer/backfintfrecords\n## My Score: 0.764","metadata":{"papermill":{"duration":6.311769,"end_time":"2021-12-09T15:57:47.872207","exception":false,"start_time":"2021-12-09T15:57:41.560438","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install -q efficientnet\n!pip install tensorflow_addons\nimport re\nimport os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nimport tensorflow as tf\nimport efficientnet.tfkeras as efn\nfrom sklearn import metrics\nfrom sklearn.model_selection import KFold, train_test_split\nfrom tensorflow.keras import backend as K\nimport tensorflow_addons as tfa\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nimport pickle\nimport json\nimport tensorflow_hub as tfhub\nfrom datetime import datetime\nimport math, re, os\nfrom matplotlib import pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nprint(\"Tensorflow version \" + tf.__version__)\nAUTO = tf.data.experimental.AUTOTUNE","metadata":{"papermill":{"duration":0.058907,"end_time":"2021-12-09T15:57:47.954148","exception":false,"start_time":"2021-12-09T15:57:47.895241","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T10:03:47.272227Z","iopub.execute_input":"2022-04-07T10:03:47.272581Z","iopub.status.idle":"2022-04-07T10:04:09.310733Z","shell.execute_reply.started":"2022-04-07T10:03:47.272471Z","shell.execute_reply":"2022-04-07T10:04:09.309096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TPU or GPU detection","metadata":{"papermill":{"duration":0.020735,"end_time":"2021-12-09T15:57:47.996437","exception":false,"start_time":"2021-12-09T15:57:47.975702","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# NEW on TPU in TensorFlow 24: shorter cross-compatible TPU/GPU/multi-GPU/cluster-GPU detection code\n\ntry: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # detect GPUs\n    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n    #strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n    #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","metadata":{"papermill":{"duration":6.323517,"end_time":"2021-12-09T15:57:54.341272","exception":false,"start_time":"2021-12-09T15:57:48.017755","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T10:04:09.312038Z","iopub.execute_input":"2022-04-07T10:04:09.312254Z","iopub.status.idle":"2022-04-07T10:04:09.330053Z","shell.execute_reply.started":"2022-04-07T10:04:09.312228Z","shell.execute_reply":"2022-04-07T10:04:09.329346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:04:09.331423Z","iopub.execute_input":"2022-04-07T10:04:09.331874Z","iopub.status.idle":"2022-04-07T10:04:09.346850Z","shell.execute_reply.started":"2022-04-07T10:04:09.331840Z","shell.execute_reply":"2022-04-07T10:04:09.345957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path(\"backfintfrecords\") # you can list the bucket with \"!gsutil ls $GCS_DS_PATH\"","metadata":{"papermill":{"duration":0.637412,"end_time":"2021-12-09T15:57:55.049507","exception":false,"start_time":"2021-12-09T15:57:54.412095","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T10:04:09.349324Z","iopub.execute_input":"2022-04-07T10:04:09.349814Z","iopub.status.idle":"2022-04-07T10:04:09.840025Z","shell.execute_reply.started":"2022-04-07T10:04:09.349779Z","shell.execute_reply":"2022-04-07T10:04:09.839364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH + '/happywhale-2022-train*.tfrec')))\ntest_files = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH + '/happywhale-2022-test*.tfrec')))\nprint(GCS_DS_PATH)\nprint(len(train_files),len(test_files),count_data_items(train_files),count_data_items(test_files))","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:04:09.841149Z","iopub.execute_input":"2022-04-07T10:04:09.841758Z","iopub.status.idle":"2022-04-07T10:04:10.385080Z","shell.execute_reply.started":"2022-04-07T10:04:09.841722Z","shell.execute_reply":"2022-04-07T10:04:10.384296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{"papermill":{"duration":0.022244,"end_time":"2021-12-09T15:57:55.096533","exception":false,"start_time":"2021-12-09T15:57:55.074289","status":"completed"},"tags":[]}},{"cell_type":"code","source":"IMAGE_SIZE = [384, 384] # At this size, a GPU will run out of memory. Use the TPU.\n                        # For GPU training, please select 224 x 224 px image size.\nEPOCHS = 30\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nFOLDS = 10\nFOLD_TO_RUN = 0\n\n\nGCS_PATH = KaggleDatasets().get_gcs_path(\"backfintfrecords\")\n\ntrain_files = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH + '/happywhale-2022-train*.tfrec')))\n\nTEST_FILENAMES = np.sort(np.array(tf.io.gfile.glob(GCS_DS_PATH + '/happywhale-2022-test*.tfrec')))\n\nTRAINING_FILENAMES = [x for i,x in enumerate(train_files) if i%FOLDS!=FOLD_TO_RUN]\nVALIDATION_FILENAMES = [x for i,x in enumerate(train_files) if i%FOLDS==FOLD_TO_RUN]\nprint(len(TRAINING_FILENAMES),len(VALIDATION_FILENAMES),count_data_items(TRAINING_FILENAMES),count_data_items(VALIDATION_FILENAMES))\n    \nN_CLASSES = 15587                                                                                                                                           # 100 - 102","metadata":{"papermill":{"duration":0.271392,"end_time":"2021-12-09T15:57:55.390978","exception":false,"start_time":"2021-12-09T15:57:55.119586","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T10:04:10.386268Z","iopub.execute_input":"2022-04-07T10:04:10.386654Z","iopub.status.idle":"2022-04-07T10:04:11.174500Z","shell.execute_reply.started":"2022-04-07T10:04:10.386624Z","shell.execute_reply":"2022-04-07T10:04:11.173405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization utilities\ndata -> pixels, nothing of much interest for the machine learning practitioner in this section.","metadata":{"papermill":{"duration":0.022816,"end_time":"2021-12-09T15:57:55.439342","exception":false,"start_time":"2021-12-09T15:57:55.416526","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef title_from_label_and_target(label, correct_label):\n    if correct_label is None:\n        return CLASSES[label], True\n    correct = (label == correct_label)\n    return \"{} [{}{}{}]\".format(CLASSES[label], 'OK' if correct else 'NO', u\"\\u2192\" if not correct else '',\n                                CLASSES[correct_label] if not correct else ''), correct\n\ndef display_one_flower(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n\ndef display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","metadata":{"papermill":{"duration":0.058906,"end_time":"2021-12-09T15:57:55.5211","exception":false,"start_time":"2021-12-09T15:57:55.462194","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T10:04:11.176539Z","iopub.execute_input":"2022-04-07T10:04:11.176863Z","iopub.status.idle":"2022-04-07T10:04:11.207784Z","shell.execute_reply.started":"2022-04-07T10:04:11.176821Z","shell.execute_reply":"2022-04-07T10:04:11.206761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Datasets","metadata":{"papermill":{"duration":0.024012,"end_time":"2021-12-09T15:57:55.570365","exception":false,"start_time":"2021-12-09T15:57:55.546353","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# data augmentation\ndef add_gaussian_noise(image):\n    # image must be scaled in [0, 1]\n    with tf.name_scope('Add_gaussian_noise'):\n        noise = tf.random.normal(shape=tf.shape(image), mean=0.0, stddev=(25)/(255), dtype=tf.float32)\n        noise_img = image + noise\n        noise_img = tf.clip_by_value(noise_img, 0.0, 1.0)\n    return noise_img\n\ndef add_gaussian_blue(image):\n    image = tfa.image.gaussian_filter2d(image)\n    return image\n\ndef random_lighting(image):\n    \"\"\"\n    Applys random augmentations related to lighting\n    \"\"\"\n    #apply random brightness\n    image = tf.image.random_brightness(image, 0.3,)\n    #apply random contrast\n    image = tf.image.random_contrast(image, 0.6, 1.4,)\n    #apply random hue\n    image = tf.image.random_hue(image, 0.002)\n    #apply random saturation\n    image = tf.image.random_saturation(image, 0.3, 1.7,)\n    return image","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:04:11.209342Z","iopub.execute_input":"2022-04-07T10:04:11.209684Z","iopub.status.idle":"2022-04-07T10:04:11.224193Z","shell.execute_reply.started":"2022-04-07T10:04:11.209640Z","shell.execute_reply":"2022-04-07T10:04:11.223525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    if tf.random.uniform([])>0.5:\n        image = tf.image.random_flip_left_right(image)\n        \n    if tf.random.uniform([])>0.5:\n        image = random_lighting(image)\n        \n    if tf.random.uniform([])>0.7:\n        image = add_gaussian_noise(image)\n        \n    if tf.random.uniform([])>0.7:\n        image = add_gaussian_blue(image)\n    return image, label  ","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:04:11.225598Z","iopub.execute_input":"2022-04-07T10:04:11.226293Z","iopub.status.idle":"2022-04-07T10:04:11.240329Z","shell.execute_reply.started":"2022-04-07T10:04:11.226251Z","shell.execute_reply":"2022-04-07T10:04:11.239412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels = 3)\n    image = tf.image.resize(image, [IMAGE_SIZE[0],IMAGE_SIZE[0]])\n    image = tf.cast(image, tf.float32) / 255.0\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"target\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['target'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"image_name\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['image_name']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset \n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\nVALIDATION_STEPS = -(-NUM_VALIDATION_IMAGES // BATCH_SIZE) # The \"-(-//)\" trick rounds up instead of down :-)\nTEST_STEPS = -(-NUM_TEST_IMAGES // BATCH_SIZE)             # The \"-(-//)\" trick rounds up instead of down :-)\nprint('Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:04:11.242072Z","iopub.execute_input":"2022-04-07T10:04:11.242744Z","iopub.status.idle":"2022-04-07T10:04:11.267909Z","shell.execute_reply.started":"2022-04-07T10:04:11.242695Z","shell.execute_reply":"2022-04-07T10:04:11.266661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset visualizations","metadata":{"papermill":{"duration":0.023077,"end_time":"2021-12-09T15:57:55.69414","exception":false,"start_time":"2021-12-09T15:57:55.671063","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# data dump\nprint(\"Training data shapes:\")\nfor image, label in get_training_dataset().take(3):\n    print(image.numpy().shape, label.numpy().shape)\nprint(\"Training data label examples:\", label.numpy())\nprint(\"Validation data shapes:\")\nfor image, label in get_validation_dataset().take(3):\n    print(image.numpy().shape, label.numpy().shape)\nprint(\"Validation data label examples:\", label.numpy())\nprint(\"Test data shapes:\")\nfor image, idnum in get_test_dataset().take(3):\n    print(image.numpy().shape, idnum.numpy().shape)\nprint(\"Test data IDs:\", idnum.numpy().astype('U')) # U=unicode string","metadata":{"papermill":{"duration":4.985826,"end_time":"2021-12-09T15:58:00.703066","exception":false,"start_time":"2021-12-09T15:57:55.71724","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T10:04:11.269314Z","iopub.execute_input":"2022-04-07T10:04:11.270182Z","iopub.status.idle":"2022-04-07T10:05:08.657307Z","shell.execute_reply.started":"2022-04-07T10:04:11.270133Z","shell.execute_reply":"2022-04-07T10:05:08.656630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 10; col = 8;\nrow = min(row,BATCH_SIZE//col)\n\nds = get_training_dataset()\n\nfor (img,label) in ds:\n#     img = sample['inp1']\n    plt.figure(figsize=(25,int(25*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.title(label[j].numpy())\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break\nprint(img.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:05:08.658704Z","iopub.execute_input":"2022-04-07T10:05:08.659619Z","iopub.status.idle":"2022-04-07T10:06:30.735764Z","shell.execute_reply.started":"2022-04-07T10:05:08.659522Z","shell.execute_reply":"2022-04-07T10:06:30.734858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 10; col = 8;\nrow = min(row,BATCH_SIZE//col)\n\nds = get_validation_dataset()\n\nfor (img,label) in ds:\n#     img = sample['inp1']\n    plt.figure(figsize=(25,int(25*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.title(label[j].numpy())\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break\nprint(img.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:06:30.738951Z","iopub.execute_input":"2022-04-07T10:06:30.739223Z","iopub.status.idle":"2022-04-07T10:06:33.184568Z","shell.execute_reply.started":"2022-04-07T10:06:30.739187Z","shell.execute_reply":"2022-04-07T10:06:33.183530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 10; col = 8;\nrow = min(row,BATCH_SIZE//col)\n\nds = get_test_dataset()\n\nfor (img,label) in ds:\n#     img = sample['inp1']\n    plt.figure(figsize=(25,int(25*row/col)))\n    for j in range(row*col):\n        plt.subplot(row,col,j+1)\n        plt.title(label[j].numpy())\n        plt.axis('off')\n        plt.imshow(img[j,])\n    plt.show()\n    break\nprint(img.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:06:33.185856Z","iopub.execute_input":"2022-04-07T10:06:33.186078Z","iopub.status.idle":"2022-04-07T10:06:35.734595Z","shell.execute_reply.started":"2022-04-07T10:06:33.186050Z","shell.execute_reply":"2022-04-07T10:06:35.733637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\nYou can select these models:  \n`swin_tiny_224`    \n`swin_small_224`  \n`swin_base_224`  \n`swin_base_384`  \n`swin_large_224`  \n`swin_large_384`  ","metadata":{"papermill":{"duration":0.117946,"end_time":"2021-12-09T15:58:05.684418","exception":false,"start_time":"2021-12-09T15:58:05.566472","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class ArcMarginProduct(tf.keras.layers.Layer):\n    '''\n    Implements large margin arc distance.\n\n    Reference:\n        https://arxiv.org/pdf/1801.07698.pdf\n        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n            blob/master/src/modeling/metric_learning.py\n    '''\n    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n                 ls_eps=0.0, **kwargs):\n\n        super(ArcMarginProduct, self).__init__(**kwargs)\n\n        self.n_classes = n_classes\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps\n        self.easy_margin = easy_margin\n        self.cos_m = tf.math.cos(m)\n        self.sin_m = tf.math.sin(m)\n        self.th = tf.math.cos(math.pi - m)\n        self.mm = tf.math.sin(math.pi - m) * m\n\n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'n_classes': self.n_classes,\n            's': self.s,\n            'm': self.m,\n            'ls_eps': self.ls_eps,\n            'easy_margin': self.easy_margin,\n        })\n        return config\n\n    def build(self, input_shape):\n        super(ArcMarginProduct, self).build(input_shape[0])\n\n        self.W = self.add_weight(\n            name='W',\n            shape=(int(input_shape[0][-1]), self.n_classes),\n            initializer='glorot_uniform',\n            dtype='float32',\n            trainable=True,\n            regularizer=None)\n\n    def call(self, inputs):\n        X, y = inputs\n        y = tf.cast(y, dtype=tf.int32)\n        cosine = tf.matmul(\n            tf.math.l2_normalize(X, axis=1),\n            tf.math.l2_normalize(self.W, axis=0)\n        )\n        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = tf.where(cosine > 0, phi, cosine)\n        else:\n            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n        one_hot = tf.cast(\n            tf.one_hot(y, depth=self.n_classes),\n            dtype=cosine.dtype\n        )\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n        return output","metadata":{"papermill":{"duration":0.140417,"end_time":"2021-12-09T15:58:05.941045","exception":false,"start_time":"2021-12-09T15:58:05.800628","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T10:06:35.735869Z","iopub.execute_input":"2022-04-07T10:06:35.736098Z","iopub.status.idle":"2022-04-07T10:06:35.754745Z","shell.execute_reply.started":"2022-04-07T10:06:35.736062Z","shell.execute_reply":"2022-04-07T10:06:35.753155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RESUME = False\nRESUME_EPOCH = None","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:06:35.756961Z","iopub.execute_input":"2022-04-07T10:06:35.758173Z","iopub.status.idle":"2022-04-07T10:06:35.769894Z","shell.execute_reply.started":"2022-04-07T10:06:35.758119Z","shell.execute_reply":"2022-04-07T10:06:35.768732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(plot=False):\n    lr_start   = 0.000001\n    lr_max     = 0.000005 * BATCH_SIZE  \n    lr_min     = 0.000001\n    lr_ramp_ep = 6\n    lr_sus_ep  = 0\n    lr_decay   = 0.9\n   \n    def lrfn(epoch):\n        if RESUME:\n            epoch = epoch + RESUME_EPOCH\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n        \n    if plot:\n        epochs = list(range(EPOCHS))\n        learning_rates = [lrfn(x) for x in epochs]\n        plt.scatter(epochs,learning_rates)\n        plt.show()\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\nget_lr_callback(plot=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:06:35.770987Z","iopub.execute_input":"2022-04-07T10:06:35.771758Z","iopub.status.idle":"2022-04-07T10:06:35.996569Z","shell.execute_reply.started":"2022-04-07T10:06:35.771724Z","shell.execute_reply":"2022-04-07T10:06:35.995605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Snapshot(tf.keras.callbacks.Callback):\n    \n    def __init__(self,fold,snapshot_epochs=[]):\n        super(Snapshot, self).__init__()\n        self.snapshot_epochs = snapshot_epochs\n        self.fold = fold\n        \n        \n    def on_epoch_end(self, epoch, logs=None):\n        # logs is a dictionary\n#         print(f\"epoch: {epoch}, train_acc: {logs['acc']}, valid_acc: {logs['val_acc']}\")\n        if epoch in self.snapshot_epochs: # your custom condition         \n            self.model.save_weights(f\"Swintranform_epoch{epoch}.h5\")\n        self.model.save_weights(f\"Swintranform_last.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:06:35.998221Z","iopub.execute_input":"2022-04-07T10:06:35.998573Z","iopub.status.idle":"2022-04-07T10:06:36.007349Z","shell.execute_reply.started":"2022-04-07T10:06:35.998528Z","shell.execute_reply":"2022-04-07T10:06:36.006173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Dropout, Conv2D, LayerNormalization, GlobalAveragePooling1D\n\nCFGS = {\n    'swin_tiny_224': dict(input_size=(224, 224), window_size=7, embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24]),\n    'swin_small_224': dict(input_size=(224, 224), window_size=7, embed_dim=96, depths=[2, 2, 18, 2], num_heads=[3, 6, 12, 24]),\n    'swin_base_224': dict(input_size=(224, 224), window_size=7, embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32]),\n    'swin_base_384': dict(input_size=(384, 384), window_size=12, embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32]),\n    'swin_large_224': dict(input_size=(224, 224), window_size=7, embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48]),\n    'swin_large_384': dict(input_size=(384, 384), window_size=12, embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48])\n}\n\n\nclass Mlp(tf.keras.layers.Layer):\n    def __init__(self, in_features, hidden_features=None, out_features=None, drop=0., prefix=''):\n        super().__init__()\n        out_features = out_features or in_features\n        hidden_features = hidden_features or in_features\n        self.fc1 = Dense(hidden_features, name=f'{prefix}/mlp/fc1')\n        self.fc2 = Dense(out_features, name=f'{prefix}/mlp/fc2')\n        self.drop = Dropout(drop)\n\n    def call(self, x):\n        x = self.fc1(x)\n        x = tf.keras.activations.gelu(x)\n        x = self.drop(x)\n        x = self.fc2(x)\n        x = self.drop(x)\n        return x\n\n\ndef window_partition(x, window_size):\n    B, H, W, C = x.get_shape().as_list()\n    x = tf.reshape(x, shape=[-1, H // window_size,\n                   window_size, W // window_size, window_size, C])\n    x = tf.transpose(x, perm=[0, 1, 3, 2, 4, 5])\n    windows = tf.reshape(x, shape=[-1, window_size, window_size, C])\n    return windows\n\n\ndef window_reverse(windows, window_size, H, W, C):\n    x = tf.reshape(windows, shape=[-1, H // window_size,\n                   W // window_size, window_size, window_size, C])\n    x = tf.transpose(x, perm=[0, 1, 3, 2, 4, 5])\n    x = tf.reshape(x, shape=[-1, H, W, C])\n    return x\n\n\nclass WindowAttention(tf.keras.layers.Layer):\n    def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0., prefix=''):\n        super().__init__()\n        self.dim = dim\n        self.window_size = window_size\n        self.num_heads = num_heads\n        head_dim = dim // num_heads\n        self.scale = qk_scale or head_dim ** -0.5\n        self.prefix = prefix\n\n        self.qkv = Dense(dim * 3, use_bias=qkv_bias,\n                         name=f'{self.prefix}/attn/qkv')\n        self.attn_drop = Dropout(attn_drop)\n        self.proj = Dense(dim, name=f'{self.prefix}/attn/proj')\n        self.proj_drop = Dropout(proj_drop)\n\n    def build(self, input_shape):\n        self.relative_position_bias_table = self.add_weight(f'{self.prefix}/attn/relative_position_bias_table',\n                                                            shape=(\n                                                                (2 * self.window_size[0] - 1) * (2 * self.window_size[1] - 1), self.num_heads),\n                                                            initializer=tf.initializers.Zeros(), trainable=True)\n\n        coords_h = np.arange(self.window_size[0])\n        coords_w = np.arange(self.window_size[1])\n        coords = np.stack(np.meshgrid(coords_h, coords_w, indexing='ij'))\n        coords_flatten = coords.reshape(2, -1)\n        relative_coords = coords_flatten[:, :,\n                                         None] - coords_flatten[:, None, :]\n        relative_coords = relative_coords.transpose([1, 2, 0])\n        relative_coords[:, :, 0] += self.window_size[0] - 1\n        relative_coords[:, :, 1] += self.window_size[1] - 1\n        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1\n        relative_position_index = relative_coords.sum(-1).astype(np.int64)\n        self.relative_position_index = tf.Variable(initial_value=tf.convert_to_tensor(\n            relative_position_index), trainable=False, name=f'{self.prefix}/attn/relative_position_index')\n        self.built = True\n\n    def call(self, x, mask=None):\n        B_, N, C = x.get_shape().as_list()\n        qkv = tf.transpose(tf.reshape(self.qkv(\n            x), shape=[-1, N, 3, self.num_heads, C // self.num_heads]), perm=[2, 0, 3, 1, 4])\n        q, k, v = qkv[0], qkv[1], qkv[2]\n\n        q = q * self.scale\n        attn = (q @ tf.transpose(k, perm=[0, 1, 3, 2]))\n        relative_position_bias = tf.gather(self.relative_position_bias_table, tf.reshape(\n            self.relative_position_index, shape=[-1]))\n        relative_position_bias = tf.reshape(relative_position_bias, shape=[\n                                            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1])\n        relative_position_bias = tf.transpose(\n            relative_position_bias, perm=[2, 0, 1])\n        attn = attn + tf.expand_dims(relative_position_bias, axis=0)\n\n        if mask is not None:\n            nW = mask.get_shape()[0]  # tf.shape(mask)[0]\n            attn = tf.reshape(attn, shape=[-1, nW, self.num_heads, N, N]) + tf.cast(\n                tf.expand_dims(tf.expand_dims(mask, axis=1), axis=0), attn.dtype)\n            attn = tf.reshape(attn, shape=[-1, self.num_heads, N, N])\n            attn = tf.nn.softmax(attn, axis=-1)\n        else:\n            attn = tf.nn.softmax(attn, axis=-1)\n\n        attn = self.attn_drop(attn)\n\n        x = tf.transpose((attn @ v), perm=[0, 2, 1, 3])\n        x = tf.reshape(x, shape=[-1, N, C])\n        x = self.proj(x)\n        x = self.proj_drop(x)\n        return x\n\n\ndef drop_path(inputs, drop_prob, is_training):\n    if (not is_training) or (drop_prob == 0.):\n        return inputs\n\n    # Compute keep_prob\n    keep_prob = 1.0 - drop_prob\n\n    # Compute drop_connect tensor\n    random_tensor = keep_prob\n    shape = (tf.shape(inputs)[0],) + (1,) * \\\n        (len(tf.shape(inputs)) - 1)\n    random_tensor += tf.random.uniform(shape, dtype=inputs.dtype)\n    binary_tensor = tf.floor(random_tensor)\n    output = tf.math.divide(inputs, keep_prob) * binary_tensor\n    return output\n\n\nclass DropPath(tf.keras.layers.Layer):\n    def __init__(self, drop_prob=None):\n        super().__init__()\n        self.drop_prob = drop_prob\n\n    def call(self, x, training=None):\n        return drop_path(x, self.drop_prob, training)\n\n\nclass SwinTransformerBlock(tf.keras.layers.Layer):\n    def __init__(self, dim, input_resolution, num_heads, window_size=7, shift_size=0, mlp_ratio=4.,\n                 qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path_prob=0., norm_layer=LayerNormalization, prefix=''):\n        super().__init__()\n        self.dim = dim\n        self.input_resolution = input_resolution\n        self.num_heads = num_heads\n        self.window_size = window_size\n        self.shift_size = shift_size\n        self.mlp_ratio = mlp_ratio\n        if min(self.input_resolution) <= self.window_size:\n            self.shift_size = 0\n            self.window_size = min(self.input_resolution)\n        assert 0 <= self.shift_size < self.window_size, \"shift_size must in 0-window_size\"\n        self.prefix = prefix\n\n        self.norm1 = norm_layer(epsilon=1e-5, name=f'{self.prefix}/norm1')\n        self.attn = WindowAttention(dim, window_size=(self.window_size, self.window_size), num_heads=num_heads,\n                                    qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop, prefix=self.prefix)\n        self.drop_path = DropPath(\n            drop_path_prob if drop_path_prob > 0. else 0.)\n        self.norm2 = norm_layer(epsilon=1e-5, name=f'{self.prefix}/norm2')\n        mlp_hidden_dim = int(dim * mlp_ratio)\n        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim,\n                       drop=drop, prefix=self.prefix)\n\n    def build(self, input_shape):\n        if self.shift_size > 0:\n            H, W = self.input_resolution\n            img_mask = np.zeros([1, H, W, 1])\n            h_slices = (slice(0, -self.window_size),\n                        slice(-self.window_size, -self.shift_size),\n                        slice(-self.shift_size, None))\n            w_slices = (slice(0, -self.window_size),\n                        slice(-self.window_size, -self.shift_size),\n                        slice(-self.shift_size, None))\n            cnt = 0\n            for h in h_slices:\n                for w in w_slices:\n                    img_mask[:, h, w, :] = cnt\n                    cnt += 1\n\n            img_mask = tf.convert_to_tensor(img_mask)\n            mask_windows = window_partition(img_mask, self.window_size)\n            mask_windows = tf.reshape(\n                mask_windows, shape=[-1, self.window_size * self.window_size])\n            attn_mask = tf.expand_dims(\n                mask_windows, axis=1) - tf.expand_dims(mask_windows, axis=2)\n            attn_mask = tf.where(attn_mask != 0, -100.0, attn_mask)\n            attn_mask = tf.where(attn_mask == 0, 0.0, attn_mask)\n            self.attn_mask = tf.Variable(\n                initial_value=attn_mask, trainable=False, name=f'{self.prefix}/attn_mask')\n        else:\n            self.attn_mask = None\n\n        self.built = True\n\n    def call(self, x):\n        H, W = self.input_resolution\n        B, L, C = x.get_shape().as_list()\n        assert L == H * W, \"input feature has wrong size\"\n\n        shortcut = x\n        x = self.norm1(x)\n        x = tf.reshape(x, shape=[-1, H, W, C])\n\n        # cyclic shift\n        if self.shift_size > 0:\n            shifted_x = tf.roll(\n                x, shift=[-self.shift_size, -self.shift_size], axis=[1, 2])\n        else:\n            shifted_x = x\n\n        # partition windows\n        x_windows = window_partition(shifted_x, self.window_size)\n        x_windows = tf.reshape(\n            x_windows, shape=[-1, self.window_size * self.window_size, C])\n\n        # W-MSA/SW-MSA\n        attn_windows = self.attn(x_windows, mask=self.attn_mask)\n\n        # merge windows\n        attn_windows = tf.reshape(\n            attn_windows, shape=[-1, self.window_size, self.window_size, C])\n        shifted_x = window_reverse(attn_windows, self.window_size, H, W, C)\n\n        # reverse cyclic shift\n        if self.shift_size > 0:\n            x = tf.roll(shifted_x, shift=[\n                        self.shift_size, self.shift_size], axis=[1, 2])\n        else:\n            x = shifted_x\n        x = tf.reshape(x, shape=[-1, H * W, C])\n\n        # FFN\n        x = shortcut + self.drop_path(x)\n        x = x + self.drop_path(self.mlp(self.norm2(x)))\n\n        return x\n\n\nclass PatchMerging(tf.keras.layers.Layer):\n    def __init__(self, input_resolution, dim, norm_layer=LayerNormalization, prefix=''):\n        super().__init__()\n        self.input_resolution = input_resolution\n        self.dim = dim\n        self.reduction = Dense(2 * dim, use_bias=False,\n                               name=f'{prefix}/downsample/reduction')\n        self.norm = norm_layer(epsilon=1e-5, name=f'{prefix}/downsample/norm')\n\n    def call(self, x):\n        H, W = self.input_resolution\n        B, L, C = x.get_shape().as_list()\n        assert L == H * W, \"input feature has wrong size\"\n        assert H % 2 == 0 and W % 2 == 0, f\"x size ({H}*{W}) are not even.\"\n\n        x = tf.reshape(x, shape=[-1, H, W, C])\n\n        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C\n        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C\n        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C\n        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C\n        x = tf.concat([x0, x1, x2, x3], axis=-1)\n        x = tf.reshape(x, shape=[-1, (H // 2) * (W // 2), 4 * C])\n\n        x = self.norm(x)\n        x = self.reduction(x)\n\n        return x\n\n\nclass BasicLayer(tf.keras.layers.Layer):\n    def __init__(self, dim, input_resolution, depth, num_heads, window_size,\n                 mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0.,\n                 drop_path_prob=0., norm_layer=LayerNormalization, downsample=None, use_checkpoint=False, prefix=''):\n        super().__init__()\n        self.dim = dim\n        self.input_resolution = input_resolution\n        self.depth = depth\n        self.use_checkpoint = use_checkpoint\n\n        # build blocks\n        self.blocks = tf.keras.Sequential([SwinTransformerBlock(dim=dim, input_resolution=input_resolution,\n                                           num_heads=num_heads, window_size=window_size,\n                                           shift_size=0 if (\n                                               i % 2 == 0) else window_size // 2,\n                                           mlp_ratio=mlp_ratio,\n                                           qkv_bias=qkv_bias, qk_scale=qk_scale,\n                                           drop=drop, attn_drop=attn_drop,\n                                           drop_path_prob=drop_path_prob[i] if isinstance(\n                                               drop_path_prob, list) else drop_path_prob,\n                                           norm_layer=norm_layer,\n                                           prefix=f'{prefix}/blocks{i}') for i in range(depth)])\n        if downsample is not None:\n            self.downsample = downsample(\n                input_resolution, dim=dim, norm_layer=norm_layer, prefix=prefix)\n        else:\n            self.downsample = None\n\n    def call(self, x):\n        x = self.blocks(x)\n\n        if self.downsample is not None:\n            x = self.downsample(x)\n        return x\n\n\nclass PatchEmbed(tf.keras.layers.Layer):\n    def __init__(self, img_size=(224, 224), patch_size=(4, 4), in_chans=3, embed_dim=96, norm_layer=None):\n        super().__init__(name='patch_embed')\n        patches_resolution = [img_size[0] //\n                              patch_size[0], img_size[1] // patch_size[1]]\n        self.img_size = img_size\n        self.patch_size = patch_size\n        self.patches_resolution = patches_resolution\n        self.num_patches = patches_resolution[0] * patches_resolution[1]\n\n        self.in_chans = in_chans\n        self.embed_dim = embed_dim\n\n        self.proj = Conv2D(embed_dim, kernel_size=patch_size,\n                           strides=patch_size, name='proj')\n        if norm_layer is not None:\n            self.norm = norm_layer(epsilon=1e-5, name='norm')\n        else:\n            self.norm = None\n\n    def call(self, x):\n        B, H, W, C = x.get_shape().as_list()\n        assert H == self.img_size[0] and W == self.img_size[1], \\\n            f\"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]}).\"\n        x = self.proj(x)\n        x = tf.reshape(\n            x, shape=[-1, (H // self.patch_size[0]) * (W // self.patch_size[0]), self.embed_dim])\n        if self.norm is not None:\n            x = self.norm(x)\n        return x\n\n\nclass SwinTransformerModel(tf.keras.Model):\n    def __init__(self, model_name='swin_tiny_patch4_window7_224', include_top=False,\n                 img_size=(224, 224), patch_size=(4, 4), in_chans=3, num_classes=1000,\n                 embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24],\n                 window_size=7, mlp_ratio=4., qkv_bias=True, qk_scale=None,\n                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1,\n                 norm_layer=LayerNormalization, ape=False, patch_norm=True,\n                 use_checkpoint=False, **kwargs):\n        super().__init__(name=model_name)\n\n        self.include_top = include_top\n\n        self.num_classes = num_classes\n        self.num_layers = len(depths)\n        self.embed_dim = embed_dim\n        self.ape = ape\n        self.patch_norm = patch_norm\n        self.num_features = int(embed_dim * 2 ** (self.num_layers - 1))\n        self.mlp_ratio = mlp_ratio\n\n        # split image into non-overlapping patches\n        self.patch_embed = PatchEmbed(\n            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim,\n            norm_layer=norm_layer if self.patch_norm else None)\n        num_patches = self.patch_embed.num_patches\n        patches_resolution = self.patch_embed.patches_resolution\n        self.patches_resolution = patches_resolution\n\n        # absolute postion embedding\n        if self.ape:\n            self.absolute_pos_embed = self.add_weight('absolute_pos_embed',\n                                                      shape=(\n                                                          1, num_patches, embed_dim),\n                                                      initializer=tf.initializers.Zeros())\n\n        self.pos_drop = Dropout(drop_rate)\n\n        # stochastic depth\n        dpr = [x for x in np.linspace(0., drop_path_rate, sum(depths))]\n\n        # build layers\n        self.basic_layers = tf.keras.Sequential([BasicLayer(dim=int(embed_dim * 2 ** i_layer),\n                                                input_resolution=(patches_resolution[0] // (2 ** i_layer),\n                                                                  patches_resolution[1] // (2 ** i_layer)),\n                                                depth=depths[i_layer],\n                                                num_heads=num_heads[i_layer],\n                                                window_size=window_size,\n                                                mlp_ratio=self.mlp_ratio,\n                                                qkv_bias=qkv_bias, qk_scale=qk_scale,\n                                                drop=drop_rate, attn_drop=attn_drop_rate,\n                                                drop_path_prob=dpr[sum(depths[:i_layer]):sum(\n                                                    depths[:i_layer + 1])],\n                                                norm_layer=norm_layer,\n                                                downsample=PatchMerging if (\n                                                    i_layer < self.num_layers - 1) else None,\n                                                use_checkpoint=use_checkpoint,\n                                                prefix=f'layers{i_layer}') for i_layer in range(self.num_layers)])\n        self.norm = norm_layer(epsilon=1e-5, name='norm')\n        self.avgpool = GlobalAveragePooling1D()\n        if self.include_top:\n            self.head = Dense(num_classes, name='head')\n        else:\n            self.head = None\n\n    def forward_features(self, x):\n        x = self.patch_embed(x)\n        if self.ape:\n            x = x + self.absolute_pos_embed\n        x = self.pos_drop(x)\n\n        x = self.basic_layers(x)\n        x = self.norm(x)\n        x = self.avgpool(x)\n        return x\n\n    def call(self, x):\n        x = self.forward_features(x)\n        if self.include_top:\n            x = self.head(x)\n        return x\n\n\ndef SwinTransformer(model_name='swin_tiny_224', num_classes=1000, include_top=True, pretrained=True, use_tpu=False, cfgs=CFGS):\n    cfg = cfgs[model_name]\n    net = SwinTransformerModel(\n        model_name=model_name, include_top=include_top, num_classes=num_classes, img_size=cfg['input_size'], window_size=cfg[\n            'window_size'], embed_dim=cfg['embed_dim'], depths=cfg['depths'], num_heads=cfg['num_heads']\n    )\n    net(tf.keras.Input(shape=(cfg['input_size'][0], cfg['input_size'][1], 3)))\n    if pretrained is True:\n        url = f'https://github.com/rishigami/Swin-Transformer-TF/releases/download/v0.1-tf-swin-weights/{model_name}.tgz'\n        pretrained_ckpt = tf.keras.utils.get_file(\n            model_name, url, untar=True)\n    else:\n        pretrained_ckpt = pretrained\n\n    if pretrained_ckpt:\n        if tf.io.gfile.isdir(pretrained_ckpt):\n            pretrained_ckpt = f'{pretrained_ckpt}/{model_name}.ckpt'\n\n        if use_tpu:\n            load_locally = tf.saved_model.LoadOptions(\n                experimental_io_device='/job:localhost')\n            net.load_weights(pretrained_ckpt, options=load_locally)\n        else:\n            net.load_weights(pretrained_ckpt)\n\n    return net","metadata":{"papermill":{"duration":96.091669,"end_time":"2021-12-09T15:59:42.150051","exception":false,"start_time":"2021-12-09T15:58:06.058382","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T10:06:36.009650Z","iopub.execute_input":"2022-04-07T10:06:36.010110Z","iopub.status.idle":"2022-04-07T10:06:36.125396Z","shell.execute_reply.started":"2022-04-07T10:06:36.010061Z","shell.execute_reply":"2022-04-07T10:06:36.124401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def freeze_BN(model):\n    # Unfreeze layers while leaving BatchNorm layers frozen\n    for layer in model.layers:\n        if not isinstance(layer, tf.keras.layers.BatchNormalization):\n            layer.trainable = True\n        else:\n            layer.trainable = False","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:06:36.126704Z","iopub.execute_input":"2022-04-07T10:06:36.126942Z","iopub.status.idle":"2022-04-07T10:06:36.132318Z","shell.execute_reply.started":"2022-04-07T10:06:36.126913Z","shell.execute_reply":"2022-04-07T10:06:36.131472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to create our EfficientNetB3 model\ndef get_model():\n\n    EMB_DIM = 512\n    N_CLASSES_MODEL = N_CLASSES\n\n    with strategy.scope():\n        inp = tf.keras.layers.Input(shape=(*IMAGE_SIZE, 3), name=\"inp1\")\n        label = tf.keras.layers.Input(shape=(), name=\"inp2\")\n\n        model_feat = SwinTransformer('swin_large_384', num_classes=N_CLASSES_MODEL, include_top=False, pretrained=True, use_tpu=True)\n\n        embed = model_feat(inp)\n        embed = tf.keras.layers.BatchNormalization()(embed) # batch norm or L2\n        embed = tf.keras.layers.Dropout(0.2)(embed)\n        embed = tf.keras.layers.Dense(EMB_DIM, name=\"dense_before_arcface\", kernel_initializer=\"he_normal\")(embed)\n#         embed = tf.keras.layers.BatchNormalization()(embed) # batch norm or L2\n\n        x = ArcMarginProduct(\n            n_classes=N_CLASSES_MODEL,\n            s=30,\n            m=0.3,\n            name=\"head/arc_margin\",\n            dtype=\"float32\"\n        )([embed, label])\n\n        output = tf.keras.layers.Softmax(dtype=\"float32\")(x)\n\n        model = tf.keras.Model(inputs=[inp, label], outputs=[output])\n        embed_model = tf.keras.Model(inputs = inp, outputs = embed)\n\n        model.compile(\n            optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3, epsilon=1e-5),\n            loss    = [ tf.keras.losses.SparseCategoricalCrossentropy()],\n            metrics = [tf.keras.metrics.SparseCategoricalAccuracy(),\n                       tf.keras.metrics.SparseTopKCategoricalAccuracy(k=5)]\n        )\n        model.summary()\n        \n        return model,embed_model","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:06:36.133761Z","iopub.execute_input":"2022-04-07T10:06:36.134167Z","iopub.status.idle":"2022-04-07T10:06:36.149923Z","shell.execute_reply.started":"2022-04-07T10:06:36.134122Z","shell.execute_reply":"2022-04-07T10:06:36.148980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"papermill":{"duration":0.490937,"end_time":"2021-12-09T15:59:43.150564","exception":false,"start_time":"2021-12-09T15:59:42.659627","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_dataset = get_training_dataset()\ntrain_dataset = train_dataset.map(lambda image, label: ({'inp1': image, 'inp2': label}, label) )\n\nvalid_dataset = get_validation_dataset()\nvalid_dataset = valid_dataset.map(lambda image, label: ({'inp1': image, 'inp2': label}, label) )\n\n# history = model.fit(train_dataset, steps_per_epoch=STEPS_PER_EPOCH, epochs=EPOCHS,\n#                     validation_data=valid_dataset, validation_steps=VALIDATION_STEPS)\nSTEPS_PER_EPOCH = count_data_items(TRAINING_FILENAMES) // BATCH_SIZE\n# Create a callback that saves the model's weights\nsv_loss = tf.keras.callbacks.ModelCheckpoint(f\"Swintrainform_loss.h5\", monitor='val_loss', verbose=0, save_best_only=True,\n                                                    save_weights_only=True, mode='min', save_freq='epoch')\ntrain_logger = tf.keras.callbacks.CSVLogger('training-log-fold-%i.h5.csv'%FOLD_TO_RUN)\n\nsnap = Snapshot(fold=FOLD_TO_RUN,snapshot_epochs=[25,26,27,28,29,30,31])","metadata":{"papermill":{"duration":646.2659,"end_time":"2021-12-09T16:10:29.883363","exception":false,"start_time":"2021-12-09T15:59:43.617463","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-04-07T10:06:36.151241Z","iopub.execute_input":"2022-04-07T10:06:36.151516Z","iopub.status.idle":"2022-04-07T10:06:36.421342Z","shell.execute_reply.started":"2022-04-07T10:06:36.151482Z","shell.execute_reply":"2022-04-07T10:06:36.420421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import backend as K","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:06:36.422864Z","iopub.execute_input":"2022-04-07T10:06:36.423509Z","iopub.status.idle":"2022-04-07T10:06:36.428361Z","shell.execute_reply.started":"2022-04-07T10:06:36.423463Z","shell.execute_reply":"2022-04-07T10:06:36.427386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # BUILD MODEL\n# K.clear_session()\n# model,embed_model = get_model()","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:07:45.929928Z","iopub.execute_input":"2022-04-07T10:07:45.930297Z","iopub.status.idle":"2022-04-07T10:07:45.935588Z","shell.execute_reply.started":"2022-04-07T10:07:45.930263Z","shell.execute_reply":"2022-04-07T10:07:45.934364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = model.fit(train_dataset,\n#                     steps_per_epoch=STEPS_PER_EPOCH,\n#                     epochs=EPOCHS,\n#                     validation_data=valid_dataset,\n#                     callbacks = [snap,get_lr_callback(),train_logger,sv_loss],\n#                     )","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:07:48.188837Z","iopub.execute_input":"2022-04-07T10:07:48.189984Z","iopub.status.idle":"2022-04-07T10:07:48.195019Z","shell.execute_reply.started":"2022-04-07T10:07:48.189904Z","shell.execute_reply":"2022-04-07T10:07:48.193825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EVALUATION","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/swin-submission/submission_764.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:07:56.901388Z","iopub.execute_input":"2022-04-07T10:07:56.902641Z","iopub.status.idle":"2022-04-07T10:07:56.963653Z","shell.execute_reply.started":"2022-04-07T10:07:56.902586Z","shell.execute_reply":"2022-04-07T10:07:56.962602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-07T10:07:57.068248Z","iopub.execute_input":"2022-04-07T10:07:57.068572Z","iopub.status.idle":"2022-04-07T10:07:57.214167Z","shell.execute_reply.started":"2022-04-07T10:07:57.068538Z","shell.execute_reply":"2022-04-07T10:07:57.213206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}