{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Flower Classification with Data Augmentations & Densenet201\nWe added and tested random contrast and bright change as data augmentation methods in this notebook.\n\nThis notebook is based on this work, thire augmentation methods is kept.\nhttps://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96"},{"metadata":{"trusted":true},"cell_type":"code","source":"''' Code structure\n1 imports\n2 def # (original)\n        decode_image\n        read_labeled_tfrecord\n        read_unlabeled_tfrecord\n        load_dataset\n        data_augment\n        get_training_dataset\n        get_validation_dataset\n        get_test_dataset\n        count_data_items\n   Data Augmentation\n        get_mat\n        transform_ori\n        lrfn\n   ANN functions\n        get_model\n        train_cross_validate\n        train_and_predict\n3 def # Modeifications, our work\n    \n4 main function\n    train_and_predict\n5 Confusion Matrix and Validation Score   '''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nprint('Start: '+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n\nimport random, re, math\nimport numpy as np, pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n\nimport tensorflow as tf, tensorflow.keras.backend as K\nprint('    Tensorflow version ' + tf.__version__)\nfrom tensorflow.keras.applications import DenseNet201\nfrom kaggle_datasets import KaggleDatasets\n\n\ndef decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0  # convert image to floats in [0, 1] range\n    image = tf.reshape(image, [*IMAGE_SIZE, 3]) # explicit size needed for TPU\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"class\": tf.io.FixedLenFeature([], tf.int64),  # shape [] means single element\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label # returns a dataset of (image, label) pairs\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n        \"id\": tf.io.FixedLenFeature([], tf.string),  # shape [] means single element\n        # class is missing, this competitions's challenge is to predict flower classes for the test dataset\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum # returns a dataset of image(s)\n\ndef load_dataset(filenames, labeled = True, ordered = False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n        \n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads = AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls = AUTO) # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    return dataset\n\ndef data_augment(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.random_flip_left_right(image)\n    return image, label   \n\ndef get_training_dataset(dataset,do_aug=True):\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    if do_aug: dataset = dataset.map(transform, num_parallel_calls=AUTO)\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(dataset):\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_test_dataset(ordered=False):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames): # V\n    # the number of data items is written in the name of the .tfrec files, i.e. flowers00-230.tfrec = 230 data items\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) for filename in filenames]\n    return np.sum(n)\n\n###############     ##############\n# # Data Augmentation # 数据增强用函数\n    # 以下代码使用GPU/TPU进行随机旋转，剪切，缩放和移位。 当图像从显示空白的边缘移开时，通过拉伸原始边缘上的颜色来填充空白。 \n    # 在下面的函数“ transform（）”中更改变量，以控制所需的扩充量。 \ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies  返回用于转换索引的3x3转换矩阵\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )   \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))\n\ndef transform_ori(image,label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted 随机旋转，剪切，缩放和移动\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n    # GET TRANSFORMATION MATRIX \n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3));#d: tf.Tensor(shape=(262144, 3), dtype=float32)  (for 512px image)        \n    return tf.reshape(d,[DIM,DIM,3]),label\n\n# 计算学习率用函数\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\n# ANN functions #神经网络用函数\ndef get_model():\n    print('    -------------get_model is running')\n    with strategy.scope():\n        rnet = DenseNet201(\n            input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3),weights='imagenet',include_top=False)\n        # trainable rnet\n        rnet.trainable = True\n        model = tf.keras.Sequential([rnet,tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(len(CLASSES), activation='softmax',dtype='float32') ])\n    model.compile(optimizer='adam',loss = 'sparse_categorical_crossentropy',metrics=['sparse_categorical_accuracy'])\n    return model\n\ndef train_cross_validate(folds = 5):\n    print('    -------------train_cross_validate is running')\n    histories = [];   models = [];\n    early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)\n    kfold = KFold(folds, shuffle = True, random_state = SEED)\n    for f, (trn_ind, val_ind) in enumerate(kfold.split(TRAINING_FILENAMES)):\n        print('    '+'#'*25);print('    ### FOLD',f+1);\n        train_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[trn_ind]['TRAINING_FILENAMES']), labeled = True)\n        val_dataset = load_dataset(list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES']), labeled = True, ordered = True)\n        model = get_model()\n        if no_training_mode:\n            print('    No training mode is on, training is skipped: no_training_mode == True')\n            models.append(model)\n            return histories, models\n        else:\n            history = model.fit(\n                get_training_dataset(train_dataset), \n                steps_per_epoch = STEPS_PER_EPOCH,epochs = EPOCHS,\n                callbacks = [lr_callback],#, early_stopping],\n                validation_data = get_validation_dataset(val_dataset),\n                verbose=1)\n            models.append(model)\n            histories.append(history)\n        if no_folds:\n            print('    No folds mode is on, the other folds is skipped: no_training_mode == no_folds')\n            return histories, models\n    return histories, models\n\ndef train_and_predict(folds = 5):\n    print('    -------------train_and_predict is running')\n    test_ds = get_test_dataset(ordered=True) \n        # since we are splitting the dataset and iterating separately on images and ids, order matters.\n    test_images_ds = test_ds.map(lambda image, idnum: image)\n    \n    # 正式训练\n    print('Start training %i folds'%folds)\n    histories, models = train_cross_validate(folds = folds)\n    \n    print('Computing predictions...')\n    # get the mean probability of the folds models 交叉验证得到平均概率\n    if no_folds: # 快速测试时仅仅只跑一次fold，而非交叉验证（[models[0]）\n        probabilities = np.average([models[0].predict(test_images_ds)], axis = 0)\n    else:\n        probabilities = np.average([models[i].predict(test_images_ds) for i in range(folds)], axis = 0)\n        \n    predictions = np.argmax(probabilities, axis=-1)\n    print('Generating submission.csv file...')\n    test_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n    test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n    np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='')\n    return histories, models\n    \n######################### Modified Functions ######################### Our work\ndef plot_history(history, label, epcohs):\n    data = {}; \n    data[label] = [0]; data[label].extend(history.history[label]) \n    data['val_' + label] = [0];  data['val_' + label].extend(history.history['val_' + label])\n    pd.DataFrame(data).plot(figsize=(8, 5))\n    plt.grid(True);plt.axis([1, epochs, 0, data[label][1]*1.5]);plt.show()\n    \ndef contrastImg_ts(img_file1,alpha,beta,gamma):  #Adjust Contrast & Brightness randomly\n    #随机调整对比度及亮度\\\n    alpha = tf.random.uniform([1],dtype='float32')*0.3+0.7;\n    gamma = (tf.random.uniform([1],dtype='float32')-0.5)*0.8;\n    beta = 1.0 - alpha;#     print(IMAGE_SIZE)\n    img2 = tf.zeros([IMAGE_SIZE[0],IMAGE_SIZE[1],3],dtype='float32');\n    print(\"原图权重=\",alpha);\n    contrasted = img_file1*alpha + img2*beta + gamma;\n    return contrasted\n    \ndef transform(image,label): # Basicly same as the original\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0];XDIM = DIM%2 #fix for size 331\n    \n    rot = 15. * tf.random.normal([1],dtype='float32');\n    shr = 5. * tf.random.normal([1],dtype='float32') ;\n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.;\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.;\n    h_shift = 16. * tf.random.normal([1],dtype='float32') ;\n    w_shift = 16. * tf.random.normal([1],dtype='float32') ;\n  \n    # GET TRANSFORMATION MATRIX \n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift);\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM );\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] );\n    z = tf.ones([DIM*DIM],dtype='int32');\n    idx = tf.stack( [x,y,z] );\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'));\n    idx2 = K.cast(idx2,dtype='int32');\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2);\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] );\n    img_changed = contrastImg_ts(image,alpha=1,beta=0,gamma=0);    # NEW\n    d = tf.gather_nd(img_changed,tf.transpose(idx3));\n    \n    return tf.reshape(d,[DIM,DIM,3]),label\n\n####   main function # 主函数\nif __name__ == \"__main__\":\n    upload_mode = True\n    # Configuration 设置 及 超参数 \n    print('Configuring: ' + time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n    if upload_mode:\n        IMAGE_SIZE =[512,512]# [331,331]#[512,512];# [192,192];# \n        EPOCHS = 15\n        FOLDS = 2;        SEED = 777;\n        # BATCH_SIZE = 16 * strategy.num_replicas_in_sync   # 在下方，此处strategy未生成\n        quick_test = False;no_training_mode = False; show_augmented_exampe = True; no_folds = False;\n    else:\n        IMAGE_SIZE = [512,512]#[192,192];\n        EPOCHS = 20\n        FOLDS = 2\n        SEED = 777\n        no_folds = True; # 跳过folds步骤\n        quick_test = False; # 加载少量数据进行训练\n        no_training_mode = False;#True; \n        show_augmented_exampe = False;#True; \n        \n    MIXED_PRECISION = False;     XLA_ACCELERATE = False\n\n    # Configurations 硬件配置\n    print('Detecting Hardware:'+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n    # Detect hardware, return appropriate distribution strategy 检测硬件，返回适当的分配策略\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()# Cluster Resolver for Google Cloud TPUs.适用于GoogleCloudTPU的集群解析器。\n        # TPU detection. No parameters necessary if TPU_NAME TPU检测。 如果TPU_NAME，则不需要任何参数\n        #    environment variable is set. On Kaggle this is always the case. 环境变量已设置。 在Kaggle上，情况总是如此。\n        print('    Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)# Connects to the given cluster.\n        tf.tpu.experimental.initialize_tpu_system(tpu)# Initialize the TPU devices.\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)# TPU distribution strategy implementation.\n    else:\n        strategy = tf.distribute.get_strategy() \n        # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\n    BATCH_SIZE = 16 * strategy.num_replicas_in_sync # \n    AUTO = tf.data.experimental.AUTOTUNE\n    print(\"    REPLICAS: \", strategy.num_replicas_in_sync) # 多线程？\n    \n\n    # Mixed Precision and/or XLA \n    # 以下布尔值可以在GPU / TPU上启用混合精度和/或XLA。 默认情况下，TPU已经使用了某种混合精度，但我们可以添加更多精度。 \n    #这些使GPU / TPU内存可以处理更大的批处理大小，并可以加快训练过程。 \n    #Nvidia V100 GPU具有特殊的Tensor核心，在启用混合精度后会被利用。 Kaggle的Nvidia P100 GPU没有Tensor Core来加速。\n    if MIXED_PRECISION:\n        print('    Mixed precision enabled:'+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n        from tensorflow.keras.mixed_precision import experimental as mixed_precision\n        if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n        else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n        mixed_precision.set_policy(policy)\n        print('    Mixed precision enabled')\n        \n    if XLA_ACCELERATE:\n        print('   Accelerated Linear Algebra:'+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n        tf.config.optimizer.set_jit(True)\n        print('    Accelerated Linear Algebra enabled')\n        \n# # Data Directories 数据路径与获取\n    # Data access # 数据路径\n    print('Accessing Data: '+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n    GCS_DS_PATH = KaggleDatasets().get_gcs_path('flower-classification-with-tpus')\n    # available image sizes :\n    GCS_PATH_SELECT = { \n        192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n        331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'}\n    print('    IMAGE_SIZE:' + str(IMAGE_SIZE)) # 图像尺寸\n    GCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n    TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec') + tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\n    # predictions on this dataset should be submitted for the competition\n    TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')\n    \n    if quick_test: # 数据截取(减少数据加速测试)\n        print('数据截取中(减少数据加速测试): '+ time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n        TRAINING_FILENAMES = TRAINING_FILENAMES[0:2];#print(TRAINING_FILENAMES);\n        TEST_FILENAMES = TEST_FILENAMES[0:1];#print(TEST_FILENAMES)\n        print('\\n    QUICK TEST MODEL!!! very little amount of data \\n')\n    else:\n        print('\\n    FULL TEST MODEL!!! ALL data\\n')\n    \n    # Classes 定义花朵类别\n    CLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n               'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n               'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n               'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n               'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n               'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n               'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n               'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n               'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n               'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n               'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102\n    \n    # 生成学习率策略\n    print('Generating Learning Rate: '+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n    # # Custom LR scheduler 学习率策略\n    # From starter [kernel][1]: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n    # TPU，GPU和CPU的学习率计划。 #使用LR加速是因为微调了预先训练的模型。# 从高LR开始会破坏预训练的权重。\n    \n    LR_START = 0.00001\n    LR_MAX = 0.00005 * strategy.num_replicas_in_sync\n    LR_MIN = 0.00001\n    LR_RAMPUP_EPOCHS = 5\n    LR_SUSTAIN_EPOCHS = 0\n    LR_EXP_DECAY = .8\n    \n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n    \n    rng = [i for i in range(25 if EPOCHS<25 else EPOCHS)]\n    y = [lrfn(x) for x in rng]\n    if show_augmented_exampe:\n        plt.plot(rng, y) \n    print(\"    Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))\n    \n    # # Dataset Functions # 获取数据\n    # From starter [kernel][1]: https://www.kaggle.com/mgornergoogle/getting-started-with-100-flowers-on-tpu\n    \n    # 求训练、验证、测试 数量\n    NUM_TRAINING_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (FOLDS-1.)/FOLDS )\n    NUM_VALIDATION_IMAGES = int( count_data_items(TRAINING_FILENAMES) * (1./FOLDS) )\n    NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\n    STEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n    print('    Dataset: {} training images, {} validation images, {} unlabeled test images'.format(NUM_TRAINING_IMAGES, NUM_VALIDATION_IMAGES, NUM_TEST_IMAGES))\n    \n    # # Display Example Augmentation 展示增强效果\n    # 以下是3个训练图像的示例，其中每个图像随机12次增强。\n    if show_augmented_exampe:\n        print('Plotting augmented exampe: '+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n        row = 3; col = 4;\n        for ex_No in range(2): # Examples\n            all_elements = get_training_dataset(load_dataset(TRAINING_FILENAMES),do_aug=False).unbatch()\n            one_element = tf.data.Dataset.from_tensors( next(iter(all_elements)) )\n            augmented_element = one_element.repeat().map(transform).batch(row*col)\n            for (img,label) in augmented_element:\n                plt.figure(figsize=(15,int(15*row/col)))\n                for j in range(row*col):\n                    plt.subplot(row,col,j+1);plt.axis('off');\n                    plt.imshow(img[j,]);plt.show()\n                break\n        \n    # run train and predict 运行训练\n    print('Start training: '+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n    histories, models = train_and_predict(folds = FOLDS) # Original code\n    \n    # Plot loss descrease history # 绘制loss下降曲线\n    epochs = EPOCHS;plot_history(histories[0], 'loss', epochs);\n\nprint('Everything End at: '+ time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Confusion Matrix and Validation Score   \nprint('Evaluating:'+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n\n# # Confusion Matrix and Validation Score 混淆矩阵 与 验证分数生成\ndef display_confusion_matrix(cmat, score, precision, recall):\n    plt.figure(figsize=(15,15))\n    ax = plt.gca()\n    ax.matshow(cmat, cmap='Reds')\n    ax.set_xticks(range(len(CLASSES)))\n    ax.set_xticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"left\", rotation_mode=\"anchor\")\n    ax.set_yticks(range(len(CLASSES)))\n    ax.set_yticklabels(CLASSES, fontdict={'fontsize': 7})\n    plt.setp(ax.get_yticklabels(), rotation=45, ha=\"right\", rotation_mode=\"anchor\")\n    titlestring = \"\"\n    if score is not None:\n        titlestring += 'f1 = {:.3f} '.format(score)\n    if precision is not None:\n        titlestring += '\\nprecision = {:.3f} '.format(precision)\n    if recall is not None:\n        titlestring += '\\nrecall = {:.3f} '.format(recall)\n    if len(titlestring) > 0:\n        ax.text(101, 1, titlestring, fontdict={'fontsize': 18, 'horizontalalignment':'right', 'verticalalignment':'top', 'color':'#804040'})\n    plt.show()\n    \nall_labels = []; all_prob = []; all_pred = []\nkfold = KFold(FOLDS, shuffle = True, random_state = SEED)\nfor j, (trn_ind, val_ind) in enumerate( kfold.split(TRAINING_FILENAMES) ):\n    print('    Inferring fold',j+1,'validation images...'+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\n    VAL_FILES = list(pd.DataFrame({'TRAINING_FILENAMES': TRAINING_FILENAMES}).loc[val_ind]['TRAINING_FILENAMES'])\n    NUM_VALIDATION_IMAGES = count_data_items(VAL_FILES)\n    cmdataset = get_validation_dataset(load_dataset(VAL_FILES, labeled = True, ordered = True))\n    images_ds = cmdataset.map(lambda image, label: image)\n    labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n    all_labels.append( next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() ) # get everything as one batch\n\n    if quick_test or no_folds:\n        prob = models[0].predict(images_ds)\n        all_prob.append( prob )\n        all_pred.append( np.argmax(prob, axis=-1) )\n        break\n    else:\n        prob = models[j].predict(images_ds)\n        all_prob.append( prob )\n        all_pred.append( np.argmax(prob, axis=-1) )\n\nprint('Calculating Scores:'+time.strftime(\"%Y-%m-%d-%H_%M_%S\", time.localtime()))\ncm_correct_labels = np.concatenate(all_labels)\ncm_probabilities = np.concatenate(all_prob)\ncm_predictions = np.concatenate(all_pred)\n# show predicted labels\nif quick_test or no_folds:\n    print(\"    Correct   labels: \", cm_correct_labels.shape, cm_correct_labels[0:10]);\n    print(\"    Predicted labels: \", cm_predictions.shape, cm_predictions[0:10]);\nelse: \n    print(\"    Correct   labels: \", cm_correct_labels.shape, cm_correct_labels);\n    print(\"    Predicted labels: \", cm_predictions.shape, cm_predictions);\n# cal cmat, precision, recall\ncmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))\nscore = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\nprecision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\nrecall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\\\n# display confusion matrix\nif not no_training_mode:\n    display_confusion_matrix(cmat, score, precision, recall)\n\nprint('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall));   ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test Area\nuseless"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}