{"cells":[{"metadata":{},"cell_type":"markdown","source":"#  Combining embeddings\n\nIn this kernel I train a model that takes as input the [embeddings](https://www.kaggle.com/cdeotte/rapids-cuml-knn-find-duplicates) that result from encoding images of the same size with different efficient nets and also meta data. This leads to a model that has access to the different features that different pretrained networks identify. This approach could be extended to different image sizes encoded with the same efficient net (and all combinations...).\n\nIn principle the predictions of models trained in this way should not (relatively speaking) be correlated with models trained using [this](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords) notebook, which I assume a lot of people used.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nimport tensorflow.keras.layers as L\n\nfrom kaggle_datasets import KaggleDatasets\n\nfrom xgboost import XGBClassifier\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import KFold\n\nimport random\nfrom functools import reduce\n\nimport scipy\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the embedded data\nDIM = 384; EFFN_LIST = [3,4,5]; BATCH_SIZE = 128\nPATH_TO_EMBEDDINGS = '../input/embeddings-extractor/'\nembed=[]; embed_ext=[]; embed_test=[];\nfor EFFN in EFFN_LIST:\n    embed += [np.load(PATH_TO_EMBEDDINGS+'embed_train_%i_%i.npy'%(DIM,EFFN))]\n    embed_ext += [np.load(PATH_TO_EMBEDDINGS+'embed_ext_%i_%i.npy'%(DIM,EFFN))]\n    embed_test += [np.load(PATH_TO_EMBEDDINGS+'embed_test_%i_%i.npy'%(DIM,EFFN))]\nnames = np.load(PATH_TO_EMBEDDINGS+'names_train.npy')\nnames_ext = np.load(PATH_TO_EMBEDDINGS+'names_ext.npy')\nnames_test = np.load(PATH_TO_EMBEDDINGS+'names_test.npy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.concatenate((oof_df['pred'].loc[names],embed),axis=1)\n\n# LOAD TRAIN AND TEST CSV\ntest = pd.read_csv( '../input/siim-isic-melanoma-classification/test.csv' ).set_index('image_name',drop=True)\ntest = test.loc[names_test].reset_index()\nprint('Test csv shape',test.shape)\n\ntrain = pd.read_csv( '../input/melanoma-%ix%i/train.csv'%(DIM,DIM) ).set_index('image_name',drop=True)\ntrain = train.loc[names].reset_index()\ntrain.target = train.target.astype('float32')\nprint('Train csv shape',train.shape)\n\ntrain_ext = pd.read_csv( '../input/isic2019-384x384/train.csv' ).set_index('image_name',drop=True)\ntrain_ext = train_ext.loc[names_ext].reset_index()\ntrain_ext.target = train_ext.target.astype('float32')\nprint('Train 2019 csv shape',train_ext.shape)\n\nprint('Displaying train.csv below...')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Concatenate and preprocess the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Everything has been ordered in the same way, so now all that needs doing is concatenation\ncat_enc = OneHotEncoder(drop='first')\n# num_enc = MinMaxScaler()\nnum_enc = StandardScaler()\nembed_enc = [MinMaxScaler()]*len(EFFN_LIST)\nnumeric_features = ['age_approx']\ncat_features = ['sex','anatom_site_general_challenge']\n\nall_train = pd.concat((train,train_ext),axis=0)\ncats = cat_enc.fit_transform(all_train[cat_features].fillna('0')).toarray()\nnums = num_enc.fit_transform(all_train[numeric_features].fillna(0))\nlabels = pd.concat((train.target.astype('int32'),train_ext.target.astype('int32')),axis=0).to_numpy()\nembed_ENC=[];embed_ext_ENC=[]; embed_encT=[]\nfor i in range(len(EFFN_LIST)):\n    embed_enc[i].fit(np.concatenate((embed[i],embed_ext[i]),axis=0))\n    embed_ENC += [embed_enc[i].transform(embed[i])]\n    embed_ext_ENC += [embed_enc[i].transform(embed_ext[i])]\n    embed_encT += [embed_enc[i].transform(embed_test[i])]\nXtrain = np.concatenate((cats,nums),axis=1)\n\ncats = cat_enc.transform(test[cat_features].fillna('0')).toarray()\nnums = num_enc.transform(test[numeric_features].fillna(0))\nXtest = np.concatenate((cats,nums),axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[np.shape(emb) for emb in embed_ENC]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for emb in embed_ENC:\n    std_all = np.std(emb,axis=1)\n    embed_std = np.mean(std_all)\n    print('Mean standard deviation is {}'.format(embed_std))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set hyperparameters","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.random.set_seed(5);\n\nweights = {0:1, 1:50}\n\nFOLDS=5\nSEED=42\nDISPLAY_PLOT = 1\nREPLICAS=1\nDISPLAY_PLOT = 1\nEPOCHS=100\nTTA=11\nbatch_size = 128\nVERBOSE=0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dim1 = embed_ENC[0].shape[1]\ndim2 = embed_ENC[1].shape[1]\ndim3 = embed_ENC[2].shape[1]\nmeta_dim = Xtrain.shape[1]\ndef build_model(ls=0.05):\n    inp1 = tf.keras.layers.Input(shape=(dim1,))\n    inp2 = tf.keras.layers.Input(shape=(dim2,))\n    inp3 = tf.keras.layers.Input(shape=(dim3,))\n    meta_inp = tf.keras.layers.Input(shape=(meta_dim,))\n    \n    x1 = L.Dropout(0.2)(inp1)\n    x1 = L.Dense(int(dim1/3), activation='relu')(x1)\n    x2 = L.Dropout(0.2)(inp2)\n    x2 = L.Dense(int(dim2/3), activation='relu')(x2)\n    x3 = L.Dropout(0.2)(inp3)\n    x3 = L.Dense(int(dim3/3), activation='relu')(x3)\n    \n    x = L.concatenate((x1,x2,x3))\n    x = L.Dropout(0.3)(x)\n    x = L.Dense(int(1024), activation='relu')(x)\n    x = L.Dropout(0.3)(x)\n    x = L.Dense(int(512), activation='relu')(x)\n    x = L.Dropout(0.3)(x)\n    x = L.Dense(int(256), activation='relu')(x)\n    x = L.Dropout(0.3)(x)\n    x = L.Dense(int(128), activation='relu')(x)\n\n    xm = L.concatenate((x,meta_inp))\n    xm = L.Dropout(0.3)(xm)\n    xm = L.Dense(128, activation='relu')(xm)\n    xm = L.Dropout(0.3)(xm)\n    xm = L.Dense(64, activation='relu')(xm)\n    xm = L.Dropout(0.3)(xm)\n    xm = L.Dense(32, activation='relu')(xm)\n    xm = L.Dropout(0.3)(xm)\n    xm = L.Dense(16, activation='relu')(xm)\n    xm = L.Dropout(0.2)(xm)\n    xm = L.Dense(1, activation='sigmoid')(xm)\n    model = tf.keras.Model(inputs=(meta_inp,inp1,inp2,inp3), outputs=xm)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.000000125* REPLICAS * batch_size)\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=ls) \n    model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train the model\n\nThis has been repurposed from [this](https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords) notebook.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr_callback(batch_size=8):\n    lr_start   = 0.000005\n    lr_max     = 0.00000125 * REPLICAS * batch_size\n#     lr_max     = 0.0000125 * REPLICAS * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 50\n    lr_sus_ep  = 0\n    lr_decay   = 0.99\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        else:\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# helper function for loading data\n\ndef data_augment(data,mean=0.0,std1=0.02):\n    gauss=tf.random.normal(tf.shape(data), mean=mean, stddev=std1,dtype=data.dtype)\n    new_data = data + gauss\n    return new_data\n\n# How to augment meta data and image embeddings?\n\ndef aug(X):\n    return tuple([data_augment(e) for e in X])\n\ndef get_dataset(X,embed,y,augment=True,repeat=True,batch=batch_size):\n    ds = tf.data.Dataset.from_tensor_slices((  tuple([X]+embed)  , y))\n    if repeat:\n        ds = ds.repeat()\n    if augment:\n        ds = ds.map(lambda elem,label: (aug(elem),label))\n    ds = ds.batch(batch)\n    return ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ids = train['image_name']\nidsO = test['image_name']\ndata_sub = Xtest\n# Default strategy for single GPU\nstrategy = tf.distribute.get_strategy()\n\n# skf = StratifiedKFold(n_splits=FOLDS,shuffle=True,random_state=SEED)\nskf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)\n\noof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = [] \npreds = np.zeros((test.shape[0],1))\npreds_all = np.zeros((test.shape[0],FOLDS))\n\nXX_data=[]\nfor i in range(len(EFFN_LIST)):\n    XX_data += [np.concatenate([embed_ENC[i],embed_ext_ENC[i]],axis=0)]\n\nfor fold,(idxT2,idxV2) in enumerate(skf.split(np.arange(15))):\n\n    idxT = train.loc[train.tfrecord.isin(idxT2)].index.values #2020 train\n    idxV = train.loc[train.tfrecord.isin(idxV2)].index.values #2020 valid\n    \n#     X = data_augment(data[idxT]); y = train.target[idxT]\n    ext_start = len(train)\n    ext_num = len(train_ext)\n    ind = np.concatenate((idxT,list(range(ext_start,ext_num+ext_start-1))))\n    X = Xtrain[ind]; y = labels[ind];\n#     Xemb = [emb[idxT] for emb in embed_ENC];\n    XX=[elem[ind] for elem in XX_data]\n    \n    X_val = Xtrain[idxV]; y_val = labels[idxV]; XX_val=[emb[idxV] for emb in embed_ENC];\n    \n    # BUILD MODEL\n    tf.keras.backend.clear_session()\n    with strategy.scope():\n        model=build_model()\n\n    # SAVE BEST MODEL EACH FOLD\n    sv = tf.keras.callbacks.ModelCheckpoint(\n        'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,\n        save_weights_only=True, mode='min', save_freq='epoch')\n\n    # Train the model\n    history = model.fit(get_dataset(X,XX,y),epochs=EPOCHS,steps_per_epoch=X.shape[0]/batch_size//REPLICAS,\n                        verbose=VERBOSE,class_weight=weights, \n#                         callbacks=[sv,get_lr_callback(batch_size=batch_size)],\n                        callbacks=[sv],\n                       validation_data=get_dataset(X_val,XX_val, y_val,augment=False,repeat=False))\n\n    model.load_weights('fold-%i.h5'%fold)\n\n    # PREDICT OOF USING TTA\n    STEPS = TTA*X_val.shape[0]/(batch_size-10)/REPLICAS\n    pred = model.predict( get_dataset(X_val,XX_val,y_val), steps=STEPS )[:TTA*X_val.shape[0]]\n    oof_pred.append( np.mean(pred.reshape((X_val.shape[0],TTA),order='F'),axis=1) )                \n\n    # GET OOF TARGETS AND NAMES\n    oof_tar.append( y_val )\n    oof_names.append( ids[idxV] )\n    oof_folds.append( np.ones_like(oof_tar[-1],dtype='int8')*fold )\n\n    STEPS = TTA*Xtest.shape[0]/(batch_size-10)/REPLICAS\n    psub = model.predict( get_dataset(Xtest,embed_encT,np.zeros(len(Xtest))), steps=STEPS  )[:TTA*Xtest.shape[0]]\n    pstore = np.mean(psub.reshape((len(Xtest),TTA),order='F'),axis=1)\n    preds[:,0] += pstore*1/FOLDS\n\n    # REPORT RESULTS\n    auc = roc_auc_score(oof_tar[-1],oof_pred[-1])\n    oof_val.append( np.max( history.history['val_auc'] ) )\n    print('#### FOLD %i OOF AUC without TTA = %.3f, with TTA = %.3f'%(fold+1,oof_val[-1],auc))\n\n    # PLOT TRAINING\n    if DISPLAY_PLOT:\n        plt.figure(figsize=(15,5))\n        plt.plot(np.arange(EPOCHS),history.history['auc'],'-o',label='Train AUC',color='#ff7f0e')\n        plt.plot(np.arange(EPOCHS),history.history['val_auc'],'-o',label='Val AUC',color='#1f77b4')\n        x = np.argmax( history.history['val_auc'] ); y = np.max( history.history['val_auc'] )\n        xdist = plt.xlim()[1] - plt.xlim()[0]; ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#1f77b4'); plt.text(x-0.03*xdist,y-0.13*ydist,'max auc\\n%.2f'%y,size=14)\n        plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)\n        plt.legend(loc=2)\n        plt2 = plt.gca().twinx()\n        plt2.plot(np.arange(EPOCHS),history.history['loss'],'-o',label='Train Loss',color='#2ca02c')\n        plt2.plot(np.arange(EPOCHS),history.history['val_loss'],'-o',label='Val Loss',color='#d62728')\n        x = np.argmin( history.history['val_loss'] ); y = np.min( history.history['val_loss'] )\n        ydist = plt.ylim()[1] - plt.ylim()[0]\n        plt.scatter(x,y,s=200,color='#d62728'); plt.text(x-0.03*xdist,y+0.05*ydist,'min loss',size=14)\n        plt.ylabel('Loss',size=14)\n        plt.legend(loc=3)\n        plt.show()  \n\n# COMPUTE OVERALL OOF AUC\noof_c = np.concatenate(oof_pred); true = np.concatenate(oof_tar);\nnames = np.concatenate(oof_names); folds = np.concatenate(oof_folds)\nauc = roc_auc_score(true,oof_c)\nprint('Overall OOF AUC with TTA = %.5f'%auc)\n\ndd_oof = pd.DataFrame(dict(image_name = names, target=true, pred = oof_c, fold=folds))\n\nsubmission = pd.DataFrame(dict(image_name=idsO, target=preds[:,0]))\nsubmission = submission.sort_values('image_name')\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}