{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Initialize Environment"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U /kaggle/input/kerasapplications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U /kaggle/input/efficientnet/efficientnet-master","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U /kaggle/input/tensorflowresnets/TensorFlow-ResNets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def is_interactive():\n    return 'runtime'    in get_ipython().config.IPKernelApp.connection_file\nIS_INTERACTIVE = is_interactive()\nprint(IS_INTERACTIVE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os \nIS_PRIVATE = len(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))!=1\nprint(IS_PRIVATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom kaggle_datasets import KaggleDatasets\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport efficientnet.tfkeras as efn\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn.metrics import accuracy_score,confusion_matrix,classification_report, precision_recall_curve\nfrom IPython.display import display\nimport gc\nimport cv2\n\nfrom tf2_resnets import models ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TFRecords Creation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef serialize_example(image,image_name):\n    feature = {\n        'image': _bytes_feature(image),\n        'image_name': _bytes_feature(image_name),\n      }\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nRESNEXT_ID = 10\nN_TFRECORDS = 20\nIMAGE_HEIGHT = 600\nIMAGE_WIDTH = 800\nos.mkdir('test_tfrecords_600')\ntest_df = pd.DataFrame(os.listdir('../input/cassava-leaf-disease-classification/test_images/'),\n                      columns=['image_name'])\ntest_df['tfr_group'] = test_df.index%N_TFRECORDS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for tfr_group in range(N_TFRECORDS):\n    df = test_df[test_df.tfr_group==tfr_group]\n    if df.shape[0]>0:\n        tfr_filename = 'test_tfrecords_600/cassava_test{}-{}.tfrec'.format(tfr_group,df.shape[0])\n        print(\"Writing\",tfr_filename)\n        with tf.io.TFRecordWriter(tfr_filename) as writer:\n            for index,row in tqdm(df.iterrows()):\n                image_name = row['image_name']\n                image_path = '../input/cassava-leaf-disease-classification/test_images/'+image_name\n                image = cv2.imread(image_path)\n                image = cv2.resize(image,(IMAGE_WIDTH,IMAGE_HEIGHT))           \n                image_shape = image.shape\n                #image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # Fix incorrect colors\n                image = cv2.imencode('.jpg', image, (cv2.IMWRITE_JPEG_QUALITY, 100))[1].tostring()\n                image_name = str.encode(image_name)\n                sample = serialize_example(image,image_name)\n                writer.write(sample)\nimage_shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nN_TFRECORDS = 20\nIMAGE_HEIGHT = 666\nIMAGE_WIDTH = 500\nos.mkdir('test_tfrecords_500')\ntest_df = pd.DataFrame(os.listdir('../input/cassava-leaf-disease-classification/test_images/'),\n                      columns=['image_name'])\ntest_df['tfr_group'] = test_df.index%N_TFRECORDS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for tfr_group in range(N_TFRECORDS):\n    df = test_df[test_df.tfr_group==tfr_group]\n    if df.shape[0]>0:\n        tfr_filename = 'test_tfrecords_500/cassava_test{}-{}.tfrec'.format(tfr_group,df.shape[0])\n        print(\"Writing\",tfr_filename)\n        with tf.io.TFRecordWriter(tfr_filename) as writer:\n            for index,row in tqdm(df.iterrows()):\n                image_name = row['image_name']\n                image_path = '../input/cassava-leaf-disease-classification/test_images/'+image_name\n                image = cv2.imread(image_path)\n                image = cv2.resize(image,(IMAGE_WIDTH,IMAGE_HEIGHT))           \n                image_shape = image.shape\n                #image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # Fix incorrect colors\n                image = cv2.imencode('.jpg', image, (cv2.IMWRITE_JPEG_QUALITY, 100))[1].tostring()\n                image_name = str.encode(image_name)\n                sample = serialize_example(image,image_name)\n                writer.write(sample)\nimage_shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Configuration\nIn order to be a proper cross validation with a meaningful overall CV score (aligned with LB score), **you need to choose the same** `IMG_SIZES`, `INC2019`, `INC2018`, and `EFF_NETS` **for each fold**. If your goal is to just run lots of experiments, then you can choose to have a different experiment in each fold. Then each fold is like a holdout validation experiment. When you find a configuration you like, you can use that configuration for all folds. \n* DEVICE - is GPU or TPU\n* SEED - a different seed produces a different triple stratified kfold split.\n* FOLDS - number of folds. Best set to 3, 5, or 15 but can be any number between 2 and 15\n* IMG_SIZES - is a Python list of length FOLDS. These are the image sizes to use each fold\n* INC2019 - This includes the new half of the 2019 competition data. The second half of the 2019 data is the comp data from 2018 plus 2017\n* INC2018 - This includes the second half of the 2019 competition data which is the comp data from 2018 plus 2017\n* BATCH_SIZES - is a list of length FOLDS. These are batch sizes for each fold. For maximum speed, it is best to use the largest batch size your GPU or TPU allows.\n* EPOCHS - is a list of length FOLDS. These are maximum epochs. Note that each fold, the best epoch model is saved and used. So if epochs is too large, it won't matter.\n* EFF_NETS - is a list of length FOLDS. These are the EfficientNets to use each fold. The number refers to the B. So a number of `0` refers to EfficientNetB0, and `1` refers to EfficientNetB1, etc.\n* WGTS - this should be `1/FOLDS` for each fold. This is the weight when ensembling the folds to predict the test set. If you want a weird ensemble, you can use different weights.\n* TTA - test time augmentation. Each test image is randomly augmented and predicted TTA times and the average prediction is used. TTA is also applied to OOF during validation."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DEVICE = \"GPU\" #or \"GPU\"\n\n# NUMBER OF FOLDS. USE 3, 5, OR 15 \nFOLDS = 5\n\nFOLD_TO_RUN = [0,1,2,3,4]\n\n# WHICH IMAGE SIZES TO LOAD EACH FOLD\n# CHOOSE 128, 192, 256, 384, 512, 768 \n\n\n# BATCH SIZE AND EPOCHS\nBATCH_SIZE = 128\nEPOCHS = 15\n\nN_WORKERS = 4\n\n\n# TEST TIME AUGMENTATION STEPS\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        print(\"Could not connect to TPU\")\n        tpu = None\n\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except _:\n            print(\"failed to initialize TPU\")\n    else:\n        DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\":\n    print(\"Using default strategy for CPU and single GPU\")\n    strategy = tf.distribute.get_strategy()\n\nif DEVICE == \"GPU\":\n    print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Step 1: Preprocess\nPreprocess has already been done and saved to TFRecords. Here we choose which size to load. We can use either 128x128, 192x192, 256x256, 384x384, 512x512, 768x768 by changing the `IMG_SIZES` variable in the preceeding code section. These TFRecords are discussed [here][1]. The advantage of using different input sizes is discussed [here][2]\n\n[1]: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/155579\n[2]: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/160147"},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = '.'\nfiles_test_600 = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test_tfrecords_600/*.tfrec')))\nfiles_test_500 = np.sort(np.array(tf.io.gfile.glob(GCS_PATH + '/test_tfrecords_500/*.tfrec')))\nprint(files_test_600)\nprint(files_test_500)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Step 2: Data Augmentation\nThis notebook uses rotation, sheer, zoom, shift augmentation first shown in this notebook [here][1] and successfully used in Melanoma comp by AgentAuers [here][2]. This notebook also uses horizontal flip, hue, saturation, contrast, brightness augmentation similar to last years winner and also similar to AgentAuers' notebook.\n\nAdditionally we can decide to use external data by changing the variables `INC2019` and `INC2018` in the preceeding code section. These variables respectively indicate whether to load last year 2019 data and/or year 2018 + 2017 data. These datasets are discussed [here][3]\n\nConsider experimenting with different augmenation and/or external data. The code to load TFRecords is taken from AgentAuers' notebook [here][2]. Thank you AgentAuers, this is great work.\n\n[1]: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96\n[2]: https://www.kaggle.com/agentauers/incredible-tpus-finetune-effnetb0-b6-at-once\n[3]: https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/164910"},{"metadata":{"trusted":true},"cell_type":"code","source":"ROT_ = 180.0\nSHR_ = 2.0\nHZOOM_ = 8.0\nWZOOM_ = 8.0\nHSHIFT_ = 8.0\nWSHIFT_ = 8.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image, DIM=256):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = ROT_ * tf.random.normal([1], dtype='float32')\n    shr = SHR_ * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / HZOOM_\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / WZOOM_\n    h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32') \n    w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_unlabeled_tfrecord(example):\n    tfrec_format = {\n        'image'                        : tf.io.FixedLenFeature([], tf.string),\n        \"image_name\": tf.io.FixedLenFeature([], tf.string)\n    }           \n    example = tf.io.parse_single_example(example, tfrec_format)\n    return example['image'], example['image_name']\n \ndef prepare_image(img, augment=True, tta=None, dim=256):    \n    img = tf.image.decode_jpeg(img, channels=3)\n    \n    if dim==600:\n        img = tf.image.crop_to_bounding_box(img, 0, 100, 600, 600)\n    else:\n        img = tf.image.crop_to_bounding_box(img, 83, 0, 500, 500)\n        \n    img = tf.image.resize(img, [dim,dim])\n    \n    img = tf.cast(img, tf.float32) / 255.0\n    \n    if tta!=None:\n        if tta[-3:]=='_lr':\n            img = tf.image.flip_left_right(img)\n            tta = tta[:-3]\n            \n        if tta=='rotate90':\n            img = tf.image.rot90(img,1)\n            \n        if tta=='rotate180':\n            img = tf.image.rot90(img,2)\n            \n        if tta=='rotate270':\n            img = tf.image.rot90(img,3)\n                      \n    img = tf.reshape(img, [dim,dim, 3])\n            \n    return img\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(filename).group(1)) \n         for filename in filenames]\n    return np.sum(n)\ncount_data_items(files_test_600),count_data_items(files_test_500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dataset(files, augment = False, tta=None, shuffle = False, repeat = False, \n                batch_size=16, dim=512):\n    \n    ds = tf.data.TFRecordDataset(files, num_parallel_reads=AUTO)\n    \n    if repeat:\n        ds = ds.repeat()\n    \n    if shuffle: \n        ds = ds.shuffle(1024*8)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n    \n    ds = ds.map(read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n   \n    \n    ds = ds.map(lambda img, imgname_or_label: (prepare_image(img, augment=augment, tta=tta, dim=dim), \n                                               imgname_or_label), \n                num_parallel_calls=AUTO)\n    \n    ds = ds.batch(batch_size * REPLICAS)\n    ds = ds.prefetch(AUTO)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Step 3: Build Model\nThis is a common model architecute. Consider experimenting with different backbones, custom heads, losses, and optimizers. Also consider inputing meta features into your CNN."},{"metadata":{"trusted":true},"cell_type":"code","source":"EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3, \n        efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]\n\ndef build_model(dim=128, ef=0):\n    inp = tf.keras.layers.Input(shape=(dim,dim,3))\n    if ef == RESNEXT_ID:\n        base = models.ResNeXt50(input_shape=(dim,dim,3),weights=None,include_top=False)\n    else:\n        base = EFNS[ef](input_shape=(dim,dim,3),weights=None,include_top=False)\n    x = base(inp)\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    x = tf.keras.layers.Dropout(0.5)(x)\n    x = tf.keras.layers.Dense(128,activation='relu')(x)\n    x = tf.keras.layers.Dropout(0.5)(x)\n    x = tf.keras.layers.Dense(5,activation='softmax')(x)\n    model = tf.keras.Model(inputs=inp,outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n    loss = tf.keras.losses.SparseCategoricalCrossentropy()\n    metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n    model.compile(optimizer=opt,loss=loss,metrics=[metrics])\n    return model\nbuild_model().summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n    lr_start   = 0.000005\n    lr_max     = 0.00000125 * REPLICAS * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Get Test IDs"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Getting test_ids')\nIMG_SIZE = 500\nNUM_TEST_IMAGES = count_data_items(files_test_500)\nds_test = get_dataset(files_test_500,augment=False,repeat=False,shuffle=False,\n                                   dim=IMG_SIZE,batch_size=BATCH_SIZE*4)\ntest_ids_ds = ds_test.map(lambda image, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U') # all in one batch\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ids.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## INFERENCE"},{"metadata":{"trusted":true},"cell_type":"code","source":"tta_counter = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = IS_INTERACTIVE \ndef generate_submission(EFF_NET,category,TTA):\n    \n    if EFF_NET == RESNEXT_ID:\n        model_root = f'../input/cassava-category-{category}/ResNext50/ResNext50/'\n    else:\n        model_root = f'../input/cassava-category-{category}/B{EFF_NET}/B{EFF_NET}/'\n    \n    if category%2==0:\n        IMG_SIZE = 600\n        files_test = files_test_600\n    elif category%2==1:\n        IMG_SIZE = 500\n        files_test = files_test_500\n    else:\n        assert 1==2, \"INVALID CATEGORY\"\n           \n    global tta_counter\n    test_predictions = []\n    for fold in range(FOLDS):\n\n        # DISPLAY FOLD INFO\n        print('#'*25); print('#### FOLD',fold+1)\n        if EFF_NET == RESNEXT_ID:\n            print('#### Image Size %i with ResNext50 and batch_size %i'%\n                  (IMG_SIZE,BATCH_SIZE*REPLICAS))\n        else:\n            print('#### Image Size %i with EfficientNet B%i and batch_size %i'%\n                  (IMG_SIZE,EFF_NET,BATCH_SIZE*REPLICAS))\n\n\n        # BUILD MODEL\n        K.clear_session()\n        with strategy.scope():\n            model = build_model(dim=IMG_SIZE,ef=EFF_NET)\n\n        print('Loading best model...')\n        model.load_weights(model_root+'fold-%i.h5'%fold)\n\n        if len(TTA)==0:\n            print('Predicting TEST without TTA...')\n            ds_test = get_dataset(files_test,augment=False,repeat=False,shuffle=False,\n                                   dim=IMG_SIZE,batch_size=BATCH_SIZE)\n            pred = model.predict(ds_test,verbose=VERBOSE,batch_size=BATCH_SIZE,\n                                 use_multiprocessing=True,workers=N_WORKERS)\n\n        if 1:\n            print('Predicting TEST with TTA...')\n            for x_ in range(1):\n                tta = TTA[tta_counter%8]\n                print(tta)\n                tta_counter+=1\n                ds_test = get_dataset(files_test,augment=False,tta=tta,repeat=False,shuffle=False,\n                                       dim=IMG_SIZE,batch_size=BATCH_SIZE)\n                if x_==0:\n                    pred = model.predict(ds_test,verbose=VERBOSE,batch_size=BATCH_SIZE,\n                                         use_multiprocessing=True,workers=N_WORKERS)/len(TTA)\n                else:\n                    pred += model.predict(ds_test,verbose=VERBOSE,batch_size=BATCH_SIZE,\n                                          use_multiprocessing=True,workers=N_WORKERS)/len(TTA)\n\n        test_pred_fold = pd.DataFrame(pred)\n        test_pred_fold['fold'] = fold\n        test_pred_fold['image_id'] = test_ids\n        test_predictions.append(test_pred_fold)\n    test_predictions = pd.concat(test_predictions)\n    test_predictions_averaged = test_predictions.groupby('image_id')[[0,1,2,3,4]].mean()\n    test_predictions_averaged = test_predictions_averaged.sort_index()\n    return test_predictions_averaged","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CASSAVA_WEIGHTS = {(1, 'B0'): 0.0,\n (1, 'B1'): 0.0,\n (3, 'ResNext50'): 0.2144082584031884,\n (4, 'B0'): 0.04051596743735366,\n (4, 'B1'): 0.0059384033956569274,\n (4, 'B2'): 0.0,\n (4, 'B3'): 0.2772041683999385,\n (4, 'B4'): 1.0,\n (4, 'B5'): 1.0,\n (4, 'ResNext50'): 1.0,\n (5, 'B3'): 0.47634532056167095,\n (5, 'B4'): 0.0,\n (1, 'B2'): 0.0,\n (6, 'B4'): 0.81263538458694,\n (7, 'B1'): 0.6091246357587939,\n (7, 'B2'): 0.8727553512599798,\n (7, 'B3'): 1.0,\n (7, 'B4'): 0.0,\n (7, 'ResNext50'): 0.0,\n (8, 'B3'): 0.36729055080178946,\n (8, 'B4'): 0.3756019549511829,\n (8, 'ResNext50'): 1.0,\n (1, 'B3'): 0.0,\n (1, 'B4'): 1.0,\n (3, 'B0'): 0.0,\n (3, 'B1'): 0.0,\n (3, 'B2'): 0.0,\n (3, 'B3'): 0.3450946319830929,\n (3, 'B4'): 0.0}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame(test_ids,columns=['image_id'])\nfor x in range(5):\n    submission[x] = 0\nsubmission = submission.set_index('image_id')\nsubmission = submission.sort_index()\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TTA = ['rotate0', 'rotate90', 'rotate180','rotate270', 'rotate0_lr', 'rotate90_lr', 'rotate180_lr','rotate270_lr']\nfor category,effnet in CASSAVA_WEIGHTS:\n    random.shuffle(TTA)\n    wt = CASSAVA_WEIGHTS[(category,effnet)]\n    if effnet == 'ResNext50':\n        EFF_NET = RESNEXT_ID\n    else:\n        EFF_NET = int(effnet[1:])\n    if wt>0.1:\n        print(category,effnet,wt)\n        try:\n            sub = generate_submission(EFF_NET,category,TTA)\n            submission += sub*wt\n        except:\n            print(\"Exception Raised\")\nsubmission","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('predictions.csv')\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -r test_tfrecords_600/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -r test_tfrecords_500/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"K.clear_session()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## VIT\n"},{"metadata":{},"cell_type":"markdown","source":"https://www.kaggle.com/zekun98/xla-vision-transformer-vit?scriptVersionId=53884787 (V19)"},{"metadata":{"trusted":true},"cell_type":"code","source":"from numba import cuda \ndevice = cuda.get_current_device()\ndevice.reset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\n\npackage_paths = [\n    '../input/pytorch-image-models/pytorch-image-models-master', #'../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\n    '../input/image-fmix/FMix-master'\n]\nfor pth in package_paths:\n    sys.path.append(pth)\nimport timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nimport glob\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom  torch.cuda.amp import autocast, GradScaler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nimport timm #from efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom sklearn.metrics import log_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import libraries\nimport os\nimport pandas as pd\nimport albumentations as albu\nimport matplotlib.pyplot as plt\nimport json\nimport seaborn as sns\nimport cv2\nimport albumentations as albu\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold, train_test_split\nfrom albumentations.pytorch import ToTensorV2\n# from efficientnet_pytorch import EfficientNet\nimport time\nimport datetime\nimport copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'fold_num': 10,\n    'seed': 719,\n    'model_arch': 'tf_efficientnet_b3_ns',\n    'img_size': 384,\n    'epochs': 32,\n    'train_bs': 32,\n    'valid_bs': 32,\n    'lr': 1e-4,\n    'num_workers': 4,\n    'accum_iter': 1, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1,\n    'device': 'cuda:0',\n    'tta': 4,\n    'used_epochs': [8],#[6,7,8,9],\n    'weights': [1,1,1,1,1]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \ndef get_img(path):\n    im_bgr = cv2.imread(path)\n    im_rgb = im_bgr[:, :, ::-1]\n    #print(im_rgb)\n    return im_rgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DataSet class\n\nclass CassavaDataset(Dataset):\n    def __init__(self,df:pd.DataFrame,imfolder:str,train:bool = True, transforms=None):\n        self.df=df\n        self.imfolder=imfolder\n        self.train=train\n        self.transforms=transforms\n        \n    def __getitem__(self,index):\n        im_path=os.path.join(self.imfolder,self.df.iloc[index]['image_id'])\n        x=cv2.imread(im_path,cv2.IMREAD_COLOR)\n        x=cv2.cvtColor(x,cv2.COLOR_BGR2RGB)\n        \n        if(self.transforms):\n            x=self.transforms(image=x)['image']\n        \n        if(self.train):\n            y=self.df.iloc[index]['label']\n            return x,y\n        else:\n            return x\n        \n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# model_name = 'efficientnet-b7'\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nclass CustomDeiT(nn.Module):\n    def __init__(self, model_name='model_name', pretrained=False):\n        super().__init__()\n        self.model = torch.hub.load('facebookresearch/deit:main', model_name, pretrained=0)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\nclass CassvaImgClassifier(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        print(self.model)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features, n_class)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n    \nclass CustomViT(nn.Module):\n    def __init__(self, model_name='', pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n        n_features = self.model.head.in_features\n        self.model.head = nn.Linear(n_features,5)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x\n# model = create_model(\n#         'deit_base_patch16_224',\n#         pretrained=False,\n#         num_classes=5,\n        \n#         drop_block_rate=None,\n#     ).to(device)\n# model=models.(pretrained=True)\n# model.fc=nn.Linear(512,5)\n# model = EfficientNet.from_pretrained(model_name, num_classes=5) \n# model=models.resnext50_32x4d()#Add Pretrained=True to use pretrained with internet enabled\n# model.fc=nn.Linear(model.fc.in_features,5)\nmodel = CustomViT(model_name='vit_base_patch16_384', pretrained=False).to(device)\ndevice = torch.device(CFG['device'])\nmodel.to(device)\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CassavaDataset(Dataset):\n    def __init__(\n        self, df, data_root, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        \n        # get labels\n        if self.output_label:\n            target = self.df.iloc[index]['label']\n          \n        path = \"{}/{}\".format(self.data_root, self.df.iloc[index]['image_id'])\n        \n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n            \n        # do label smoothing\n        if self.output_label == True:\n            return img, target\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from albumentations.pytorch import ToTensorV2\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout,RandomCrop, ShiftScaleRotate, CenterCrop, Resize\n)\n\ndef get_inference_transforms(CFG):\n    return Compose([\n            RandomCrop(512,512),\n            Resize(CFG['img_size'], CFG['img_size']),\n#            Transpose(p=0.5),\n#            HorizontalFlip(p=0.5),\n#            VerticalFlip(p=0.5),\n#             HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n#             RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nfrom tqdm import tqdm\ndef inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)   #output = model(input)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all\nif __name__ == '__main__':\n     # for training only, need nightly build pytorch\n\n    seed_everything(CFG['seed'])\n    \n#     folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train.shape[0]), train.label.values)\n    tst_preds = []\n        \n    if 1:\n        # we'll train fold 0 first\n\n\n        test = pd.DataFrame()\n        test['image_id'] = list(os.listdir('../input/cassava-leaf-disease-classification/test_images/'))\n        test_ds = CassavaDataset(test, '../input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms(CFG), output_label=False)\n\n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n        )\n\n        device = torch.device(CFG['device'])\n#         if CFG['model_arch'] == 'resnext50_32x4d':\n#             model = CustomResNext().to(device)\n#         if 'efficientnet' in CFG['model_arch']:\n#             model = CassvaImgClassifier(CFG['model_arch'], train.label.nunique()).to(device)\n\n        #for epoch in range(CFG['epochs']-3):\n        for (i,path) in enumerate(glob.glob('../input/vit5fold/*.pth')):    \n            model.load_state_dict(torch.load(path)['model'])\n            \n            with torch.no_grad():\n                for _ in range(CFG['tta']):\n                    #val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                    tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\n        #val_preds = np.mean(val_preds, axis=0)         \n        #print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n       #print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n        del model\n        torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tst_preds = np.mean(tst_preds, axis=0) \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred = pd.DataFrame(tst_preds)\ntest_pred['image_id'] = test.image_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred = test_pred.set_index('image_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_pred.to_csv('vit_predictions.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Public Resnext"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # Directory settings\n# # ====================================================\n# import os\n\n# OUTPUT_DIR = './'\n# MODEL_DIR = '../input/cassava-resnext50-32x4d-weights/'\n# if not os.path.exists(OUTPUT_DIR):\n#     os.makedirs(OUTPUT_DIR)\n    \n# TRAIN_PATH = '../input/cassava-leaf-disease-classification/train_images'\n# TEST_PATH = '../input/cassava-leaf-disease-classification/test_images'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # CFG\n# # ====================================================\n# class CFG:\n#     debug=False\n#     num_workers=8\n#     model_name='resnext50_32x4d'\n#     size=512\n#     batch_size=32\n#     seed=2020\n#     target_size=5\n#     target_col='label'\n#     n_fold=5\n#     trn_fold=[0, 1, 2, 3, 4]\n#     inference=True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # Library\n# # ====================================================\n# import sys\n# sys.path.append('../input/pytorch-image-models/pytorch-image-models-master')\n\n# import os\n# import math\n# import time\n# import random\n# import shutil\n# from pathlib import Path\n# from contextlib import contextmanager\n# from collections import defaultdict, Counter\n\n# import scipy as sp\n# import numpy as np\n# import pandas as pd\n\n# from sklearn import preprocessing\n# from sklearn.metrics import accuracy_score\n# from sklearn.model_selection import StratifiedKFold\n\n# from tqdm.auto import tqdm\n# from functools import partial\n\n# import cv2\n# from PIL import Image\n\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n# from torch.optim import Adam, SGD\n# import torchvision.models as models\n# from torch.nn.parameter import Parameter\n# from torch.utils.data import DataLoader, Dataset\n# from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\n# import albumentations as A\n# from albumentations.pytorch import ToTensorV2\n\n# import timm\n\n# import warnings \n# warnings.filterwarnings('ignore')\n\n# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # Utils\n# # ====================================================\n# def get_score(y_true, y_pred):\n#     return accuracy_score(y_true, y_pred)\n\n\n# @contextmanager\n# def timer(name):\n#     t0 = time.time()\n#     LOGGER.info(f'[{name}] start')\n#     yield\n#     LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\n# def init_logger(log_file=OUTPUT_DIR+'inference.log'):\n#     from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n#     logger = getLogger(__name__)\n#     logger.setLevel(INFO)\n#     handler1 = StreamHandler()\n#     handler1.setFormatter(Formatter(\"%(message)s\"))\n#     handler2 = FileHandler(filename=log_file)\n#     handler2.setFormatter(Formatter(\"%(message)s\"))\n#     logger.addHandler(handler1)\n#     logger.addHandler(handler2)\n#     return logger\n\n# #LOGGER = init_logger()\n\n# def seed_torch(seed=42):\n#     random.seed(seed)\n#     os.environ['PYTHONHASHSEED'] = str(seed)\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed(seed)\n#     torch.backends.cudnn.deterministic = True\n\n# seed_torch(seed=CFG.seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test = pd.read_csv('../input/cassava-leaf-disease-classification/sample_submission.csv')\n# test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # Dataset\n# # ====================================================\n# class TestDataset(Dataset):\n#     def __init__(self, df, transform=None):\n#         self.df = df\n#         self.file_names = df['image_id'].values\n#         self.transform = transform\n        \n#     def __len__(self):\n#         return len(self.df)\n\n#     def __getitem__(self, idx):\n#         file_name = self.file_names[idx]\n#         file_path = f'{TEST_PATH}/{file_name}'\n#         image = cv2.imread(file_path)\n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n#         if self.transform:\n#             augmented = self.transform(image=image)\n#             image = augmented['image']\n#         return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # Transforms\n# # ====================================================\n# def get_transforms(*, data):\n#     if data == 'valid':\n#         return A.Compose([\n#             A.Resize(CFG.size, CFG.size),\n#             A.Normalize(\n#                 mean=[0.485, 0.456, 0.406],\n#                 std=[0.229, 0.224, 0.225],\n#             ),\n#             ToTensorV2(),\n#         ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # MODEL\n# # ====================================================\n# class CustomResNext(nn.Module):\n#     def __init__(self, model_name='resnext50_32x4d', pretrained=False):\n#         super().__init__()\n#         self.model = timm.create_model(model_name, pretrained=pretrained)\n#         n_features = self.model.fc.in_features\n#         self.model.fc = nn.Linear(n_features, CFG.target_size)\n\n#     def forward(self, x):\n#         x = self.model(x)\n#         return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # Helper functions\n# # ====================================================\n# def load_state(model_path):\n#     model = CustomResNext(CFG.model_name, pretrained=False)\n#     try:  # single GPU model_file\n#         model.load_state_dict(torch.load(model_path)['model'], strict=True)\n#         state_dict = torch.load(model_path)['model']\n#     except:  # multi GPU model_file\n#         state_dict = torch.load(model_path)['model']\n#         state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}\n\n#     return state_dict\n\n\n# def inference(model, states, test_loader, device):\n#     model.to(device)\n#     tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n#     probs = []\n#     for i, (images) in tk0:\n#         images = images.to(device)\n#         avg_preds = []\n#         for state in states:\n#             model.load_state_dict(state)\n#             model.eval()\n#             with torch.no_grad():\n#                 y_preds = model(images)\n#             avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n#         avg_preds = np.mean(avg_preds, axis=0)\n#         probs.append(avg_preds)\n#     probs = np.concatenate(probs)\n#     return probs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ====================================================\n# # inference\n# # ====================================================\n# model = CustomResNext(CFG.model_name, pretrained=False)\n# states = [load_state(MODEL_DIR+f'{CFG.model_name}_fold{fold}.pth') for fold in CFG.trn_fold]\n# test_dataset = TestDataset(test, transform=get_transforms(data='valid'))\n# test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False, \n#                          num_workers=CFG.num_workers, pin_memory=True)\n# resNext50_pred = inference(model, states, test_loader, device)\n# # # submission\n# # test['label'] = predictions.argmax(1)\n# # test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)\n# # test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# resNext50_sub = pd.DataFrame(resNext50_pred)\n# resNext50_sub['image_id'] = test.image_id\n# resNext50_sub = resNext50_sub.set_index('image_id')\n# resNext50_sub = resNext50_sub.sort_index()\n# resNext50_sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# resNext50_sub.to_csv('public_resnext_predictions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred1 = pd.read_csv('./predictions.csv',index_col=0).sort_index()\n# pred2 = pd.read_csv('./public_resnext_predictions.csv',index_col=0).sort_index()\npred3 = pd.read_csv('./vit_predictions.csv',index_col=0).sort_index()\npred1 = pred1.div(pred1.sum(axis=1),axis=0)\n# pred2 = pred2.div(pred2.sum(axis=1),axis=0)\npred3 = pred3.div(pred3.sum(axis=1),axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pred2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = 0.8*pred1 + 0.2*pred3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['label'] = submission.idxmax(axis=1)\nsubmission = submission.reset_index()\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[['image_id','label']].to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}