{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# BiLinear EfficientNetB7 Focal Loss+ Label Smoothing","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport tensorflow as tf, tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import optimizers\nfrom kaggle_datasets import KaggleDatasets","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:21.267888Z","iopub.execute_input":"2021-05-26T16:10:21.268324Z","iopub.status.idle":"2021-05-26T16:10:27.441306Z","shell.execute_reply.started":"2021-05-26T16:10:21.268224Z","shell.execute_reply":"2021-05-26T16:10:27.440367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#TPU Configurations\n\ntry: # detect TPUs\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # detect GPUs\n    strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('plant-pathology-2021-fgvc8')\nAUTO = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(GCS_DS_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:27.442698Z","iopub.execute_input":"2021-05-26T16:10:27.442970Z","iopub.status.idle":"2021-05-26T16:10:33.085124Z","shell.execute_reply.started":"2021-05-26T16:10:27.442945Z","shell.execute_reply":"2021-05-26T16:10:33.084045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 8 * REPLICAS           # 8 * 8 = 64\nWARMUP_EPOCHS = 3\nWARMUP_LEARN_RATE = 1e-3 * REPLICAS  # 0.001 * 8 = 0.008\nEPOCHS = 30\nLEARN_RATE = 3e-4 * REPLICAS         # 0.0003 * 8 = 0.0024\nIMG_SIZE_h = 600\nIMG_SIZE_w = 600\nCHANNELS = 3\nN_CLASSES = 6\nPATIENCE = 5","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:33.086913Z","iopub.execute_input":"2021-05-26T16:10:33.087203Z","iopub.status.idle":"2021-05-26T16:10:33.092069Z","shell.execute_reply.started":"2021-05-26T16:10:33.087175Z","shell.execute_reply":"2021-05-26T16:10:33.091105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom sklearn.preprocessing import MultiLabelBinarizer\n\n# 构造样本数据项\n# 标签文件目录\npath = '../input/fgvc8-train-label/'\n\ntrain = pd.read_csv(path+'train.csv')\ntrain.pop('No')\ntrain_files = train['image']\n# 删除image列\ntrain.pop('image')\n\n# Let Us Find Out the Different Types of Classes(Diseases)\nprint(train['labels'].value_counts())\n\n# Frequency of Each Class\nplt.figure(figsize=(20,12))\nlabels = sns.barplot(train.labels.value_counts().index,train.labels.value_counts())\nfor item in labels.get_xticklabels():\n    item.set_rotation(45)\n    \n# 将空格分割的文本标签转化为文本标签列表\ntrain['labels'] = train['labels'].apply(lambda s: s.split(' '))\nprint(train[:10])\n\nmlb = MultiLabelBinarizer()\ny_train = mlb.fit_transform(train['labels']).astype('float32')\nprint(y_train[:10])\nprint(mlb.classes_)\n\n# 构造样本图像文件列表\nroot = 'train_images'\nimages_paths = [(os.path.join(GCS_DS_PATH, root, train_file)) for train_file in train_files]","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:33.093481Z","iopub.execute_input":"2021-05-26T16:10:33.093972Z","iopub.status.idle":"2021-05-26T16:10:33.876087Z","shell.execute_reply.started":"2021-05-26T16:10:33.093937Z","shell.execute_reply":"2021-05-26T16:10:33.875231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 训练集、校验集分割\nfrom sklearn.model_selection import train_test_split\nx_train,x_val,y_train,y_val = train_test_split(images_paths, y_train, test_size=0.2, shuffle=True) ","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:33.877243Z","iopub.execute_input":"2021-05-26T16:10:33.877705Z","iopub.status.idle":"2021-05-26T16:10:33.947825Z","shell.execute_reply.started":"2021-05-26T16:10:33.877665Z","shell.execute_reply":"2021-05-26T16:10:33.947011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 数据增强","metadata":{}},{"cell_type":"code","source":"import math\n\n# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\ndef transform_rotation(image, height, rotation):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = rotation * tf.random.uniform([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape(tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image, height, shear):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = shear * tf.random.uniform([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape(tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3])    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shift(image, height, h_shift, w_shift):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly shifted\n    DIM = height\n    XDIM = DIM%2 #fix for size 331\n    \n    height_shift = h_shift * tf.random.uniform([1],dtype='float32') \n    width_shift = w_shift * tf.random.uniform([1],dtype='float32') \n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n        \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape(tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3])\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shift_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:33.948972Z","iopub.execute_input":"2021-05-26T16:10:33.949400Z","iopub.status.idle":"2021-05-26T16:10:33.978409Z","shell.execute_reply.started":"2021-05-26T16:10:33.949358Z","shell.execute_reply":"2021-05-26T16:10:33.977043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cutmix(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with cutmix applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = N_CLASSES\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO CUTMIX WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.int32)\n        # CHOOSE RANDOM IMAGE TO CUTMIX WITH\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        b = tf.random.uniform([],0,1) # this is beta dist with alpha=1.0\n        WIDTH = tf.cast( DIM * tf.math.sqrt(1-b),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # MAKE CUTMIX IMAGE\n        one = image[j,ya:yb,0:xa,:]\n        two = image[k,ya:yb,xa:xb,:]\n        three = image[j,ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)\n        imgs.append(img)\n        # MAKE CUTMIX LABEL\n        a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mixup(image, label, PROBABILITY = 1.0):\n    # input image - is a batch of images of size [n,dim,dim,3] not a single image of [dim,dim,3]\n    # output - a batch of images with mixup applied\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 104\n    \n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        # DO MIXUP WITH PROBABILITY DEFINED ABOVE\n        P = tf.cast( tf.random.uniform([],0,1)<=PROBABILITY, tf.float32)\n        # CHOOSE RANDOM\n        k = tf.cast( tf.random.uniform([],0,AUG_BATCH),tf.int32)\n        a = tf.random.uniform([],0,1)*P # this is beta dist with alpha=1.0\n        # MAKE MIXUP IMAGE\n        img1 = image[j,]\n        img2 = image[k,]\n        imgs.append((1-a)*img1 + a*img2)\n        # MAKE CUTMIX LABEL\n        if len(label.shape)==1:\n            lab1 = tf.one_hot(label[j],CLASSES)\n            lab2 = tf.one_hot(label[k],CLASSES)\n        else:\n            lab1 = label[j,]\n            lab2 = label[k,]\n        labs.append((1-a)*lab1 + a*lab2)\n            \n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image2 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label2 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image2,label2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform(image,label):\n    # THIS FUNCTION APPLIES BOTH CUTMIX AND MIXUP\n    DIM = IMAGE_SIZE[0]\n    CLASSES = 104\n    SWITCH = 0.5\n    CUTMIX_PROB = 0.666\n    MIXUP_PROB = 0.666\n    # FOR SWITCH PERCENT OF TIME WE DO CUTMIX AND (1-SWITCH) WE DO MIXUP\n    image2, label2 = cutmix(image, label, CUTMIX_PROB)\n    image3, label3 = mixup(image, label, MIXUP_PROB)\n    imgs = []; labs = []\n    for j in range(AUG_BATCH):\n        P = tf.cast( tf.random.uniform([],0,1)<=SWITCH, tf.float32)\n        imgs.append(P*image2[j,]+(1-P)*image3[j,])\n        labs.append(P*label2[j,]+(1-P)*label3[j,])\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR (maybe use Python typing instead?)\n    image4 = tf.reshape(tf.stack(imgs),(AUG_BATCH,DIM,DIM,3))\n    label4 = tf.reshape(tf.stack(labs),(AUG_BATCH,CLASSES))\n    return image4,label4","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 图像解码\ndef decode_image(filename, label=None, image_size=(IMG_SIZE_h, IMG_SIZE_w)):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, image_size)\n    \n    #convert to numpy and do some cv2 staff mb?\n    \n    if label is None:\n        return image\n    else:\n        return image, label\n\n# 数据增强\ndef data_augment(image, label=None):\n    p_rotation = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_spatial = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_rotate = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_pixel = tf.random.uniform([], 0, 1.0, dtype=tf.float32)    \n    p_shear = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_shift = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    p_crop = tf.random.uniform([], 0, 1.0, dtype=tf.float32)\n    \n    \n    # Flips\n    if p_spatial >= .2:\n        image = tf.image.random_flip_left_right(image)\n        image = tf.image.random_flip_up_down(image)\n        \n    # Rotates\n    if p_rotate > .75:\n        image = tf.image.rot90(image, k=3) # rotate 270º\n    elif p_rotate > .5:\n        image = tf.image.rot90(image, k=2) # rotate 180º\n    elif p_rotate > .25:\n        image = tf.image.rot90(image, k=1) # rotate 90º\n    \n    if p_rotation >= .3: # Rotation\n        image = transform_rotation(image, height=IMG_SIZE_h, rotation=45.)\n    if p_shift >= .3: # Shift\n        image = transform_shift(image, height=IMG_SIZE_h, h_shift=15., w_shift=15.)\n    if p_shear >= .3: # Shear\n        image = transform_shear(image, height=IMG_SIZE_h, shear=20.)\n        \n    # Crops\n    if p_crop > .4:\n        crop_size = tf.random.uniform([], int(IMG_SIZE_h*.7), IMG_SIZE_h, dtype=tf.int32)\n        image = tf.image.random_crop(image, size=[crop_size, crop_size, CHANNELS])\n    elif p_crop > .7:\n        if p_crop > .9:\n            image = tf.image.central_crop(image, central_fraction=.7)\n        elif p_crop > .8:\n            image = tf.image.central_crop(image, central_fraction=.8)\n        else:\n            image = tf.image.central_crop(image, central_fraction=.9)\n            \n    image = tf.image.resize(image, size=[IMG_SIZE_h, IMG_SIZE_w])\n        \n    # Pixel-level transforms\n    if p_pixel >= .2:\n        if p_pixel >= .8:\n            image = tf.image.random_saturation(image, lower=0, upper=2)\n        elif p_pixel >= .6:\n            image = tf.image.random_contrast(image, lower=.8, upper=2)\n        elif p_pixel >= .4:\n            image = tf.image.random_brightness(image, max_delta=.2)\n        else:\n            image = tf.image.adjust_gamma(image, gamma=.6)\n\n    if label is None:\n        return image\n    else:\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:33.980126Z","iopub.execute_input":"2021-05-26T16:10:33.980441Z","iopub.status.idle":"2021-05-26T16:10:34.000150Z","shell.execute_reply.started":"2021-05-26T16:10:33.980413Z","shell.execute_reply":"2021-05-26T16:10:33.999117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 构造训练集\ntrain_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .map(data_augment, num_parallel_calls=AUTO)\n    .repeat()\n    .batch(BATCH_SIZE)\n    .map(transform, num_parallel_calls=AUTO)\n    .unbatch()\n    .shuffle(10000)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n    )","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:34.003179Z","iopub.execute_input":"2021-05-26T16:10:34.003749Z","iopub.status.idle":"2021-05-26T16:10:36.172196Z","shell.execute_reply.started":"2021-05-26T16:10:34.003709Z","shell.execute_reply":"2021-05-26T16:10:36.171264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 构造校验集\nval_dataset = (tf.data.Dataset\n               .from_tensor_slices((x_val,y_val))\n               .map(decode_image,num_parallel_calls=AUTO)\n               .batch(BATCH_SIZE)\n               .cache()\n               .prefetch(AUTO)\n              )","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:36.173794Z","iopub.execute_input":"2021-05-26T16:10:36.174089Z","iopub.status.idle":"2021-05-26T16:10:36.213615Z","shell.execute_reply.started":"2021-05-26T16:10:36.174061Z","shell.execute_reply":"2021-05-26T16:10:36.212598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications import efficientnet\nimport tensorflow as tf, tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import Dense,GlobalAveragePooling2D, Input\nfrom tensorflow.keras.models import Model,Sequential\nfrom tensorflow.keras import optimizers\nfrom keras.utils.vis_utils import plot_model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:36.214749Z","iopub.execute_input":"2021-05-26T16:10:36.215038Z","iopub.status.idle":"2021-05-26T16:10:36.280908Z","shell.execute_reply.started":"2021-05-26T16:10:36.214995Z","shell.execute_reply":"2021-05-26T16:10:36.279921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam\n\ndef get_model():\n    \n    input_tensor = Input(shape=(IMG_SIZE_h, IMG_SIZE_w, CHANNELS))\n    \n    base_model = efficientnet.EfficientNetB7(weights='/kaggle/input/tfkerasefficientnetimagenetnotop/efficientnetb7_notop.h5', \n                                         include_top=False, \n                                         input_tensor=input_tensor,\n                                         input_shape=(IMG_SIZE_h, IMG_SIZE_w, CHANNELS))\n    base_model.trainable = False\n    model = tf.keras.Sequential([\n        base_model,\n        GlobalAveragePooling2D(),\n        Dense(512, activation='relu'),\n        Dense(16, activation='relu'),\n        Dense(N_CLASSES, activation='sigmoid')\n    ])\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:36.282180Z","iopub.execute_input":"2021-05-26T16:10:36.282479Z","iopub.status.idle":"2021-05-26T16:10:36.288830Z","shell.execute_reply.started":"2021-05-26T16:10:36.282449Z","shell.execute_reply":"2021-05-26T16:10:36.287839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 分类层热身训练","metadata":{}},{"cell_type":"code","source":"import tensorflow_addons as tfa\nfrom tensorflow.keras.optimizers import Adam\n\n\n\nopt = Adam(learning_rate=WARMUP_LEARN_RATE)\n\nwith strategy.scope():\n    model = get_model()\n    f1 = tfa.metrics.F1Score(num_classes=N_CLASSES, average='macro')\n\nmodel.compile(optimizer=opt, loss='binary_crossentropy', metrics=[f1])","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:10:36.290009Z","iopub.execute_input":"2021-05-26T16:10:36.290485Z","iopub.status.idle":"2021-05-26T16:11:20.020483Z","shell.execute_reply.started":"2021-05-26T16:10:36.290450Z","shell.execute_reply":"2021-05-26T16:11:20.019493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tensorflow.keras.utils import plot_model\n# plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:11:20.021974Z","iopub.execute_input":"2021-05-26T16:11:20.022368Z","iopub.status.idle":"2021-05-26T16:11:20.026105Z","shell.execute_reply.started":"2021-05-26T16:11:20.022328Z","shell.execute_reply":"2021-05-26T16:11:20.025120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_dataset,\n                    steps_per_epoch=y_train.shape[0]//BATCH_SIZE,\n                    epochs=WARMUP_EPOCHS,\n                    verbose=1,\n                    validation_data=val_dataset\n                    )","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:11:20.027445Z","iopub.execute_input":"2021-05-26T16:11:20.027992Z","iopub.status.idle":"2021-05-26T16:41:30.654223Z","shell.execute_reply.started":"2021-05-26T16:11:20.027951Z","shell.execute_reply":"2021-05-26T16:41:30.652730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 微调训练所有网络层","metadata":{}},{"cell_type":"markdown","source":"### 学习速率调度","metadata":{}},{"cell_type":"code","source":"LR_START = 0.0000001\nLR_MIN = 0.0000001\nLR_MAX = LEARN_RATE\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nsns.set(style='whitegrid')\nfig, ax = plt.subplots(figsize=(20, 6))\nplt.plot(rng, y)\n\nprint(f'{EPOCHS} total epochs and {y_train.shape[0]//BATCH_SIZE} steps per epoch')\nprint(f'Learning rate schedule: {y[0]:.3g} to {max(y):.3g} to {y[-1]:.3g}')\n","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:41:30.656412Z","iopub.execute_input":"2021-05-26T16:41:30.656720Z","iopub.status.idle":"2021-05-26T16:41:30.895626Z","shell.execute_reply.started":"2021-05-26T16:41:30.656675Z","shell.execute_reply":"2021-05-26T16:41:30.894665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 允许训练model的所有层\nfor layer in model.layers:\n    layer.trainable = True","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:41:30.897174Z","iopub.execute_input":"2021-05-26T16:41:30.897579Z","iopub.status.idle":"2021-05-26T16:41:30.930718Z","shell.execute_reply.started":"2021-05-26T16:41:30.897518Z","shell.execute_reply":"2021-05-26T16:41:30.929884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\n\n\nsave_best_model = ModelCheckpoint(filepath=\"EfficientNetB7_Best_Model.h5\",\n                                  monitor='val_f1_score', verbose=1,\n                                  save_best_only=True,\n                                  save_weights_only=False,\n                                  mode='max',\n                                  save_freq='epoch')\n\nes = EarlyStopping(monitor='val_loss', \n                   mode='min', \n                   patience=PATIENCE, \n                   restore_best_weights=True, verbose=1)\nlr_callback = LearningRateScheduler(lrfn, verbose=1)\n\ncallback_list = [save_best_model, es, lr_callback]\nopt = Adam(learning_rate=LEARN_RATE)\nmodel.compile(optimizer=opt, loss='binary_crossentropy', metrics=[f1])\n\nhistory = model.fit(train_dataset,\n                    steps_per_epoch=y_train.shape[0]//BATCH_SIZE,\n                    callbacks=callback_list,\n                    epochs=EPOCHS,\n                    verbose=1,\n                    validation_data=val_dataset\n                    )\n#it will take some time to start training\n# 将训练得到的模型保存到文件\nprint(\"[信息] 保存模型...\")\nmodel.save(\"EfficientNetB7_Last_Model.h5\", overwrite=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T16:41:30.932079Z","iopub.execute_input":"2021-05-26T16:41:30.932449Z","iopub.status.idle":"2021-05-26T20:30:35.172169Z","shell.execute_reply.started":"2021-05-26T16:41:30.932409Z","shell.execute_reply":"2021-05-26T20:30:35.171028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline \n\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\n\n\nprint ('Matplotlib version: ', mpl.__version__) # >= 2.0.0\n\nval_f1 = history.history['val_f1_score']\nf1 = history.history['f1_score']\nepochs = range(len(f1))\n\ndf_val_f1_score = pd.DataFrame(val_f1, columns = ['val_f1']) \ndf_f1_score = pd.DataFrame(f1, columns = ['f1'])\n\ndf_val_f1_score.to_csv('val_f1.csv')\ndf_f1_score.to_csv('f1.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-26T20:30:35.174740Z","iopub.execute_input":"2021-05-26T20:30:35.175047Z","iopub.status.idle":"2021-05-26T20:30:35.196178Z","shell.execute_reply.started":"2021-05-26T20:30:35.175018Z","shell.execute_reply":"2021-05-26T20:30:35.195219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(12,4)) # set the size that you'd like (width, height)\nplt.title('F1 Score')\nplt.ylabel('f1 score')\nplt.xlabel('Epochs')\nplt.plot(epochs,val_f1,label='Validation F1 Score')\nplt.plot(epochs, f1,label='Training F1 Score')\nplt.legend()\nplt.figure()\nplt.savefig('F1.pdf')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T20:30:35.197322Z","iopub.execute_input":"2021-05-26T20:30:35.197658Z","iopub.status.idle":"2021-05-26T20:30:35.476341Z","shell.execute_reply.started":"2021-05-26T20:30:35.197629Z","shell.execute_reply":"2021-05-26T20:30:35.475578Z"},"trusted":true},"execution_count":null,"outputs":[]}]}