{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Thanks to https://www.kaggle.com/xhlulu/ranzcr-efficientnet-tpu-training","metadata":{"papermill":{"duration":0.013548,"end_time":"2021-06-20T15:56:08.499616","exception":false,"start_time":"2021-06-20T15:56:08.486068","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install efficientnet -q","metadata":{"papermill":{"duration":9.494355,"end_time":"2021-06-20T15:56:18.00655","exception":false,"start_time":"2021-06-20T15:56:08.512195","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:00:26.450704Z","iopub.execute_input":"2021-06-20T22:00:26.451112Z","iopub.status.idle":"2021-06-20T22:00:35.476906Z","shell.execute_reply.started":"2021-06-20T22:00:26.451031Z","shell.execute_reply":"2021-06-20T22:00:35.475530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nimport efficientnet.tfkeras as efn\nimport numpy as np\nimport pandas as pd\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport albumentations as A\nimport math\nimport tensorflow.keras.backend as K\n\nimport gc","metadata":{"papermill":{"duration":8.29465,"end_time":"2021-06-20T15:56:26.316489","exception":false,"start_time":"2021-06-20T15:56:18.021839","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:00:35.479406Z","iopub.execute_input":"2021-06-20T22:00:35.479844Z","iopub.status.idle":"2021-06-20T22:00:43.517823Z","shell.execute_reply.started":"2021-06-20T22:00:35.479795Z","shell.execute_reply":"2021-06-20T22:00:43.516686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def auto_select_accelerator():\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n        print(\"Running on TPU:\", tpu.master())\n    except ValueError:\n        strategy = tf.distribute.get_strategy()\n    print(f\"Running on {strategy.num_replicas_in_sync} replicas\")\n    \n    return strategy","metadata":{"papermill":{"duration":0.022012,"end_time":"2021-06-20T15:56:26.351333","exception":false,"start_time":"2021-06-20T15:56:26.329321","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:00:43.520054Z","iopub.execute_input":"2021-06-20T22:00:43.520353Z","iopub.status.idle":"2021-06-20T22:00:43.526352Z","shell.execute_reply.started":"2021-06-20T22:00:43.520324Z","shell.execute_reply":"2021-06-20T22:00:43.525283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMSIZE = (224, 240, 260, 300, 380, 456, 528, 600)\nIMS = 6\n\nimg_size = IMSIZE[IMS]\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n\nCOMPETITION_NAME = \"siimcovid19-512-img-png-600-study-png\"\nstrategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 16\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(COMPETITION_NAME)\n\n#cutout\nPROBABILITY = 0.75\nCT = 8\nSZ = 0.1\nDIM = img_size\n\n\n# transformations\nROT_    = 12.0\nSHR_    = 0.0\nHZOOM_  = 0.0\nWZOOM_  = 0.0\nHSHIFT_ = 0.0\nWSHIFT_ = 0.0\n\n#bri, contrast\ncont = (0.8, 1.2)\nbri  =  0.1","metadata":{"papermill":{"duration":5.980314,"end_time":"2021-06-20T15:56:32.343946","exception":false,"start_time":"2021-06-20T15:56:26.363632","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:00:43.527797Z","iopub.execute_input":"2021-06-20T22:00:43.528239Z","iopub.status.idle":"2021-06-20T22:00:49.302013Z","shell.execute_reply.started":"2021-06-20T22:00:43.528192Z","shell.execute_reply":"2021-06-20T22:00:49.300946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dropout(image, DIM\n            =img_size, PROBABILITY = 0.75, CT = 8, SZ = 0.1):\n    # input - one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n\n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1) < PROBABILITY, tf.int32)\n    if (P == 0)|(CT == 0)|(SZ == 0): return image\n\n    for k in range( CT ):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*DIM,tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM,y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM,x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3]) \n        three = image[ya:yb,xb:DIM,:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM,:,:]],axis=0)\n\n    # RESHAPE HACK SO TPU COMPILER KNOWS SHAPE OF OUTPUT TENSOR \n    image = tf.reshape(image,[DIM,DIM,3])\n    return image\n\ndef get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, DIM=(img_size, img_size)):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    \n    # fixed for non-square image thanks to Chris Deotte\n    \n    if DIM[0]!=DIM[1]:\n        pad = (DIM[0]-DIM[1])//2\n        image = tf.pad(image, [[0, 0], [pad, pad+1],[0, 0]])\n        \n    NEW_DIM = DIM[0]\n    \n    XDIM = NEW_DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(NEW_DIM//2, -NEW_DIM//2,-1), NEW_DIM)\n    y   = tf.tile(tf.range(-NEW_DIM//2, NEW_DIM//2), [NEW_DIM])\n    z   = tf.ones([NEW_DIM*NEW_DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -NEW_DIM//2+XDIM+1, NEW_DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([NEW_DIM//2-idx2[0,], NEW_DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n    \n    if DIM[0]!=DIM[1]:\n        image = tf.reshape(d,[NEW_DIM, NEW_DIM,3])\n        image = image[:, pad:DIM[1]+pad,:]\n    image = tf.reshape(image, [*DIM, 3])\n        \n    return image","metadata":{"papermill":{"duration":0.04561,"end_time":"2021-06-20T15:56:32.403407","exception":false,"start_time":"2021-06-20T15:56:32.357797","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:00:49.303608Z","iopub.execute_input":"2021-06-20T22:00:49.304033Z","iopub.status.idle":"2021-06-20T22:00:49.332942Z","shell.execute_reply.started":"2021-06-20T22:00:49.303984Z","shell.execute_reply":"2021-06-20T22:00:49.331933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_decoder(with_labels=True, target_size=(256, 256), ext='jpg'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.image.resize(img, target_size)\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), label\n    \n    return decode_with_labels if with_labels else decode\n\ndef build_augmenter(with_labels=True):\n    data_augmentation = tf.keras.Sequential([\n        tf.keras.layers.experimental.preprocessing.RandomZoom(height_factor=(-0.6, 0.6),width_factor=(-0.6, 0.6),fill_mode=\"constant\")\n    ])\n    def augment(img):\n        img = tf.image.random_flip_left_right(img)\n        img = tf.image.random_brightness(img, 0.1)\n        img = dropout(img)\n        img = transform(img)\n        #img = tf.image.random_flip_up_down(img)\n        img = tf.squeeze(data_augmentation(tf.expand_dims(img,0), training=True), axis=0)\n        return img\n    \n    def augment_with_labels(img, label):\n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, bsize=128, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\"):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    dset = tf.data.Dataset.from_tensor_slices(slices)\n    dset = dset.map(decode_fn, num_parallel_calls=AUTO)\n    dset = dset.cache(cache_dir) if cache else dset\n    dset = dset.map(augment_fn, num_parallel_calls=AUTO) if augment else dset\n    dset = dset.repeat() if repeat else dset\n    dset = dset.shuffle(shuffle) if shuffle else dset\n    dset = dset.batch(bsize).prefetch(AUTO)\n    \n    return dset","metadata":{"papermill":{"duration":0.032524,"end_time":"2021-06-20T15:56:32.448929","exception":false,"start_time":"2021-06-20T15:56:32.416405","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:00:49.334298Z","iopub.execute_input":"2021-06-20T22:00:49.334671Z","iopub.status.idle":"2021-06-20T22:00:49.352441Z","shell.execute_reply.started":"2021-06-20T22:00:49.334640Z","shell.execute_reply":"2021-06-20T22:00:49.351669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COMPETITION_NAME = \"siimcovid19-512-img-png-600-study-png\"\n\nstrategy = auto_select_accelerator()\nBATCH_SIZE = strategy.num_replicas_in_sync * 16\nGCS_DS_PATH = KaggleDatasets().get_gcs_path(COMPETITION_NAME)","metadata":{"papermill":{"duration":8.139157,"end_time":"2021-06-20T15:56:40.600761","exception":false,"start_time":"2021-06-20T15:56:32.461604","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:00:49.353848Z","iopub.execute_input":"2021-06-20T22:00:49.354313Z","iopub.status.idle":"2021-06-20T22:00:57.203942Z","shell.execute_reply.started":"2021-06-20T22:00:49.354275Z","shell.execute_reply":"2021-06-20T22:00:57.203121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_dir = f\"/kaggle/input/{COMPETITION_NAME}/\"\ndf = pd.read_csv('../input/siim-covid19-detection/train_study_level.csv')\nlabel_cols = df.columns[1:5]","metadata":{"papermill":{"duration":0.037952,"end_time":"2021-06-20T15:56:40.652279","exception":false,"start_time":"2021-06-20T15:56:40.614327","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:00:57.206207Z","iopub.execute_input":"2021-06-20T22:00:57.206771Z","iopub.status.idle":"2021-06-20T22:00:57.230994Z","shell.execute_reply.started":"2021-06-20T22:00:57.206725Z","shell.execute_reply":"2021-06-20T22:00:57.230151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#gkf  = GroupKFold(n_splits = 5)\ngkf = StratifiedKFold(n_splits = 5)\ndf['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(X=df, y=df[label_cols].values.argmax(1), groups = df.id.tolist())):\n    df.loc[val_idx, 'fold'] = fold","metadata":{"papermill":{"duration":0.039398,"end_time":"2021-06-20T15:56:40.704778","exception":false,"start_time":"2021-06-20T15:56:40.66538","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:00:57.232451Z","iopub.execute_input":"2021-06-20T22:00:57.233042Z","iopub.status.idle":"2021-06-20T22:00:57.255144Z","shell.execute_reply.started":"2021-06-20T22:00:57.232997Z","shell.execute_reply":"2021-06-20T22:00:57.254182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntf.keras.backend.clear_session()\n\nvalidation_auc_best = []\nvalidation_loss_best = []\n\nfor i in range(5):\n    \n    valid_paths = GCS_DS_PATH + '/study/study/' + df[df['fold'] == i]['id'] + '.png' #\"/train/\"\n    train_paths = GCS_DS_PATH + '/study/study/' + df[df['fold'] != i]['id'] + '.png' #\"/train/\" \n    valid_labels = df[df['fold'] == i][label_cols].values\n    train_labels = df[df['fold'] != i][label_cols].values\n\n    valid_paths = GCS_DS_PATH + '/study/' + df[df['fold'] == i]['id'] + '.png' #\"/train/\"\n    train_paths = GCS_DS_PATH + '/study/' + df[df['fold'] != i]['id'] + '.png' #\"/train/\" \n    valid_labels = df[df['fold'] == i][label_cols].values\n    train_labels = df[df['fold'] != i][label_cols].values\n\n    decoder = build_decoder(with_labels=True, target_size=(IMSIZE[IMS], IMSIZE[IMS]), ext='png')\n    test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[IMS], IMSIZE[IMS]),ext='png')\n\n    train_dataset = build_dataset(\n        train_paths, train_labels, bsize=BATCH_SIZE, decode_fn=decoder\n    )\n\n    valid_dataset = build_dataset(\n        valid_paths, valid_labels, bsize=BATCH_SIZE, decode_fn=decoder,\n        repeat=False, shuffle=False, augment=False\n    )\n    try:\n        n_labels = train_labels.shape[1]\n    except:\n        n_labels = 1\n        \n        \n    \n\n    with strategy.scope():\n        \n        #img_adjust_layer = tf.keras.layers.Lambda(lambda data: tf.keras.applications.xception.preprocess_input(tf.cast(data, tf.float32)),\n        #                                                                                     input_shape=[img_size, img_size, 3])\n\n        pretrained_model = tf.keras.applications.xception.Xception(include_top=False, weights='imagenet')\n        pretrained_model.trainable = True\n    \n    \n        model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Dense(n_labels, activation='softmax')])\n        \n        \n        model.compile(\n        optimizer=tf.keras.optimizers.Adam(0.0001),\n        loss='categorical_crossentropy',\n        metrics=[tf.keras.metrics.AUC(multi_label=True)])\n\n        model.summary()\n\n\n    steps_per_epoch = train_paths.shape[0] // BATCH_SIZE\n    checkpoint = tf.keras.callbacks.ModelCheckpoint(\n        f'model{i}.h5', save_best_only=True, monitor='val_loss', mode='min')\n    lr_reducer = tf.keras.callbacks.ReduceLROnPlateau(factor = 0.1,\n        monitor=\"val_loss\", patience=2, min_lr=1e-6, mode='min')\n\n    history = model.fit(\n        train_dataset, \n        epochs = 40,\n        verbose=1,\n        callbacks=[checkpoint, lr_reducer],\n        steps_per_epoch=steps_per_epoch,\n        validation_data=valid_dataset)\n\n    hist_df = pd.DataFrame(history.history)\n    hist_df.to_csv(f'history{i}.csv')\n    \n    validation_auc_best.append(hist_df['val_auc'].max())\n    validation_loss_best.append(hist_df['val_loss'].min())\n    \n    del model\n    tf.keras.backend.clear_session()\n    strategy = auto_select_accelerator()","metadata":{"papermill":{"duration":7618.850155,"end_time":"2021-06-20T18:03:39.568131","exception":false,"start_time":"2021-06-20T15:56:40.717976","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:00:57.256600Z","iopub.execute_input":"2021-06-20T22:00:57.257026Z","iopub.status.idle":"2021-06-20T22:10:04.161420Z","shell.execute_reply.started":"2021-06-20T22:00:57.256981Z","shell.execute_reply":"2021-06-20T22:10:04.157507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Mean val_auc :' + str(np.array(validation_auc_best).mean()))\nprint('Std val_auc :' + str(np.array(validation_auc_best).std()))\nprint('----------------------------------')\nprint('Mean val_loss :' + str(np.array(validation_loss_best).mean()))\nprint('Std val_loss :' + str(np.array(validation_loss_best).std()))\n","metadata":{"papermill":{"duration":1.612834,"end_time":"2021-06-20T18:03:42.835997","exception":false,"start_time":"2021-06-20T18:03:41.223163","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-06-20T22:10:04.162404Z","iopub.status.idle":"2021-06-20T22:10:04.162837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":1.598558,"end_time":"2021-06-20T18:03:49.269066","exception":false,"start_time":"2021-06-20T18:03:47.670508","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}