{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This notebook is based [Triple Stratified KFold with TFRecords](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords).","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:18.294763Z","iopub.execute_input":"2021-07-04T21:29:18.295334Z","iopub.status.idle":"2021-07-04T21:29:28.118497Z","shell.execute_reply.started":"2021-07-04T21:29:18.295215Z","shell.execute_reply":"2021-07-04T21:29:28.117388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:28.121617Z","iopub.execute_input":"2021-07-04T21:29:28.121945Z","iopub.status.idle":"2021-07-04T21:29:36.395118Z","shell.execute_reply.started":"2021-07-04T21:29:28.121906Z","shell.execute_reply":"2021-07-04T21:29:36.393524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\nDEVICE = \"TPU\" #or \"GPU\"\n\n# USE DIFFERENT SEED FOR DIFFERENT STRATIFIED KFOLD\nSEED = 42\n\n\n# NUMBER OF FOLDS. USE 2, 4, 5, 10 OR 20 \nFOLDS = 2 if DEBUG else 5\n\nIMG_SIZES = [(129,65)]*FOLDS\n# BATCH SIZE AND EPOCHS\nBATCH_SIZES = [64]*FOLDS\nEPOCHS = [1 if DEBUG else 20]*FOLDS\n\n# WHICH EFFICIENTNET B? TO USE\nEFF_NETS = [0]*FOLDS\n\n# WEIGHTS FOR FOLD MODELS WHEN PREDICTING TEST\nWGTS = [1/FOLDS]*FOLDS\n\n# TEST TIME AUGMENTATION STEPS\nTTA = 2\n# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 1 if DEBUG else 2\nDISPLAY_PLOT = True","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:36.397074Z","iopub.execute_input":"2021-07-04T21:29:36.397394Z","iopub.status.idle":"2021-07-04T21:29:36.404166Z","shell.execute_reply.started":"2021-07-04T21:29:36.397361Z","shell.execute_reply":"2021-07-04T21:29:36.402949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dropout\nPROBABILITY = 0.75\nCT          = 4\nSZ          = 0.2","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:36.405706Z","iopub.execute_input":"2021-07-04T21:29:36.406272Z","iopub.status.idle":"2021-07-04T21:29:36.423928Z","shell.execute_reply.started":"2021-07-04T21:29:36.406225Z","shell.execute_reply":"2021-07-04T21:29:36.422950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:36.425352Z","iopub.execute_input":"2021-07-04T21:29:36.425881Z","iopub.status.idle":"2021-07-04T21:29:36.452413Z","shell.execute_reply.started":"2021-07-04T21:29:36.425844Z","shell.execute_reply":"2021-07-04T21:29:36.451201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS; GCS_PATH3 = [None]*FOLDS; GCS_PATH4 = [None]*FOLDS\nfor i in range(FOLDS):\n    GCS_PATH[i] = KaggleDatasets().get_gcs_path('g2net-tfrecord-spectrogram')\nfiles_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))\nfiles_test  = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')))","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:36.454208Z","iopub.execute_input":"2021-07-04T21:29:36.454595Z","iopub.status.idle":"2021-07-04T21:29:39.519027Z","shell.execute_reply.started":"2021-07-04T21:29:36.454558Z","shell.execute_reply":"2021-07-04T21:29:39.517588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dropout(image,DIM=IMG_SIZES[0], PROBABILITY = 0.6, CT = 5, SZ = 0.1):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): \n        return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*min(DIM),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3], dtype = image.dtype) \n        three = image[ya:yb,xb:DIM[1],:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0) \n\n    image = tf.reshape(image,[*DIM,3])\n    return image","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:39.520669Z","iopub.execute_input":"2021-07-04T21:29:39.521008Z","iopub.status.idle":"2021-07-04T21:29:39.534316Z","shell.execute_reply.started":"2021-07-04T21:29:39.520977Z","shell.execute_reply":"2021-07-04T21:29:39.533095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_labeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n        'target'                       : tf.io.FixedLenFeature([], tf.int64)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    \n    \n    #image = tf.io.decode_raw(example['image'],tf.float32)\n    \n    image = example['image']\n    \n    return image, example['target']\n\n\ndef read_unlabeled_tfrecord(example, return_image_name):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        'image_name'                   : tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, tfrec_format)\n    #image = tf.io.decode_raw(example['image'],tf.float32)\n    image = example['image']\n    return image, example['image_name'] if return_image_name else 0\n\n \ndef prepare_image(img, augment=True, dim=[129, 65]):    \n    img = tf.image.decode_jpeg(img, channels=3)\n    #image = tf.io.decode_raw(img, tf.float32)\n    img = tf.cast(img, tf.float32) / 255.0\n    \n    if augment:\n        #img = transform(img,DIM=dim)\n        img = tf.image.random_flip_left_right(img)\n        #img = tf.image.random_hue(img, 0.01)\n        #img = tf.image.random_saturation(img, 0.7, 1.3)\n        #img = tf.image.random_contrast(img, 0.8, 1.2)\n        #img = tf.image.random_brightness(img, 0.1)\n                      \n    img = tf.reshape(img, [*dim, 3])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:39.535752Z","iopub.execute_input":"2021-07-04T21:29:39.536086Z","iopub.status.idle":"2021-07-04T21:29:39.551140Z","shell.execute_reply.started":"2021-07-04T21:29:39.536045Z","shell.execute_reply":"2021-07-04T21:29:39.549699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(files, augment = False, shuffle = False, repeat = False, \n                labeled=True, return_image_names=True, batch_size=16, dim=256):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    ds = ds.cache()\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n        \n    if labeled: \n        ds = ds.map(read_labeled_tfrecord, num_parallel_calls=AUTO)\n    else:\n        ds = ds.map(lambda example: read_unlabeled_tfrecord(example, return_image_names), \n                    num_parallel_calls=AUTO)      \n    \n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, dim=dim), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    if labeled and augment:\n        ds = ds.map(lambda img, label: (dropout(img, DIM=dim, PROBABILITY = PROBABILITY, CT = CT, SZ = SZ), label),\n                    num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size * REPLICAS)\n    ds = ds.prefetch(AUTO)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:39.553452Z","iopub.execute_input":"2021-07-04T21:29:39.553834Z","iopub.status.idle":"2021-07-04T21:29:39.571920Z","shell.execute_reply.started":"2021-07-04T21:29:39.553803Z","shell.execute_reply":"2021-07-04T21:29:39.570798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6]\n\ndef build_model(dim=[129,65] , ef=0):\n    inp = tf.keras.layers.Input(shape=(dim[0],dim[1],3))\n    base = EFNS[ef](input_shape=(dim[0],dim[1],3),weights='imagenet',include_top=False)\n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp,outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.0001) \n    model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:39.573740Z","iopub.execute_input":"2021-07-04T21:29:39.574322Z","iopub.status.idle":"2021-07-04T21:29:39.591789Z","shell.execute_reply.started":"2021-07-04T21:29:39.574286Z","shell.execute_reply":"2021-07-04T21:29:39.590521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n    lr_start   = 0.00005\n    lr_max     = 0.001#0.00000125 * REPLICAS * batch_size\n    lr_min     = 0.00001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.75\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:39.593438Z","iopub.execute_input":"2021-07-04T21:29:39.594146Z","iopub.status.idle":"2021-07-04T21:29:39.606089Z","shell.execute_reply.started":"2021-07-04T21:29:39.594103Z","shell.execute_reply":"2021-07-04T21:29:39.604744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((count_data_items(files_test),1))\n\nfor fold,(idxT,idxV) in enumerate(skf.split(np.arange(20))):\n    \n    # DISPLAY FOLD INFO\n    if DEVICE=='TPU':\n        if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n    print('#'*25); print('#### FOLD',fold+1)\n    print('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n          (IMG_SIZES[fold][0],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))\n    \n    # CREATE TRAIN AND VALIDATION SUBSETS\n    files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])\n\n    np.random.shuffle(files_train); print('#'*25)\n    files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])\n\n    files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))\n    \n    train_images = count_data_items(files_train)\n    val_images   = count_data_items(files_valid)\n    test_images   = count_data_items(files_test)\n    \n    print('#### Training: %i | Validation: %i| Test: %i'%(train_images,val_images,test_images))\n    \n    # BUILD MODEL\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])\n        \n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n   \n    # TRAIN\n    print('Training...')\n    history = model.fit(\n        get_dataset(files_train, augment=True, shuffle=True, repeat=True,\n                dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]), \n        epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])], \n        steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,\n        validation_data=get_dataset(files_valid,augment=False,shuffle=False,\n                repeat=False,dim=IMG_SIZES[fold]), #class_weight = {0:1,1:2},\n        verbose=VERBOSE\n    )\n    \n    print('Loading best model...')\n    model.load_weights('fold-%i.h5'%fold)\n    \n    # PREDICT OOF USING TTA\n    print('Predicting OOF with TTA...')\n    ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,\n            repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n    ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS\n    pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,] \n    oof_pred.append( np.mean(pred.reshape((ct_valid,TTA),order='F'),axis=1) )                 \n    #oof_pred.append(model.predict(get_dataset(files_valid,dim=IMG_SIZES[fold]),verbose=1))\n    \n    # GET OOF TARGETS AND NAMES\n    ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n            labeled=True, return_image_names=True)\n    oof_tar.append( np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]) )\n    oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n    ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],\n                labeled=False, return_image_names=True)\n    oof_names.append( np.array([img_name.numpy().decode(\"utf-8\") for img, img_name in iter(ds.unbatch())]))\n    \n    # PREDICT TEST USING TTA\n    print('Predicting Test with TTA...')\n    ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,\n            repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)\n    ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS\n    pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test] \n    preds[:,0] += np.mean(pred.reshape((ct_test,TTA),order='F'),axis=1) * WGTS[fold]\n    \n    # REPORT RESULTS\n    print('calculating AUC score...')\n    auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n    oof_val.append(np.max( history.history['val_auc'] ))\n    print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n    \n    # PLOT TRAINING\n    if DISPLAY_PLOT:\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n        plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n        x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n        plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.title('FOLD %i - Image Size %i, EfficientNet B%i,'%\n                (fold+1,IMG_SIZES[fold][0],EFF_NETS[fold]),size=18)\n        plt.legend(loc=3)\n        plt.show()  ","metadata":{"execution":{"iopub.status.busy":"2021-07-04T21:29:39.609752Z","iopub.execute_input":"2021-07-04T21:29:39.610181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# COMPUTE OVERALL OOF AUC\noof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);\nnames = np.concatenate(oof_names); folds = np.concatenate(oof_folds)\nauc = roc_auc_score(true,oof)\nprint('Overall OOF AUC with TTA = %.3f'%auc)\n\n# SAVE OOF TO DISK\ndf_oof = pd.DataFrame(dict(\n    image_name = names, target=true, pred = oof, fold=folds))\ndf_oof.to_csv('oof.csv',index=False)\ndf_oof.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = get_dataset(files_test, augment=False, repeat=False, dim=IMG_SIZES[fold],\n                 labeled=False, return_image_names=True)\n\nimage_names = np.array([img_name.numpy().decode(\"utf-8\") \n                        for img, img_name in iter(ds.unbatch())])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(dict(id=image_names, target=preds[:,0]))\nsubmission = submission.sort_values('id') \nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(submission.target,bins=100)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}