{"cells":[{"metadata":{},"cell_type":"markdown","source":"Adverserial Validation is a technique to determine how different two distribution is. The embeddings were generated using Chris Notebook.\nhttps://www.kaggle.com/cdeotte/rapids-cuml-knn-find-duplicates","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading the Embeddings","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_embeddings = np.load('../input/cnn-embeddings-generator/embed_train_256_0.npy')\ntest_embeddings = np.load('../input/cnn-embeddings-generator/embed_test_256_0.npy')\nexternal_train_embeddings = np.load('../input/cnn-embeddings-generator/embed_ext_2019_256_0.npy')\nexternal_train_embeddings_18 = np.load('../input/cnn-embeddings-generator/embed_ext_2018_256_0.npy')\ntrain_names = np.load('../input/cnn-embeddings-generator/names_train.npy')\ntest_names = np.load('../input/cnn-embeddings-generator/names_test.npy')\nexternal_train_names = np.load('../input/cnn-embeddings-generator/names_ext_2019.npy')\ntrain_labels = np.load('../input/cnn-embeddings-generator/labels_train.npy')\nexternal_train_labels = np.load('../input/cnn-embeddings-generator/labels_ext_2019.npy')\nexternal_train_labels_18 = np.load('../input/cnn-embeddings-generator/labels_ext_2018.npy')\ntrain_embeddings.shape,test_embeddings.shape,external_train_embeddings.shape,train_names.shape,test_names.shape,external_train_names.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(dim=1280,lr=0.001):\n    inp = tf.keras.layers.Input(shape=(None,dim))\n    x = tf.keras.layers.Dense(1,activation='sigmoid')(inp)\n    model = tf.keras.Model(inputs=inp,outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=lr)\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05) \n    model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n    return model\n\ndef build_model1(dim=1280,lr=0.001):\n    inp = tf.keras.layers.Input(shape=(None,dim))\n    x = tf.keras.layers.Dense(300,activation='sigmoid')(inp)\n    x = tf.keras.layers.Dense(1,activation='sigmoid')(x)\n    model = tf.keras.Model(inputs=inp,outputs=x)\n    opt = tf.keras.optimizers.Adam(learning_rate=lr)\n    loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05) \n    model.compile(optimizer=opt,loss=loss,metrics=['AUC'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Adverserial Validation: Train vs Test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import classification_report,accuracy_score,f1_score,roc_auc_score\nskf = StratifiedKFold(n_splits=5,shuffle=True)\n\nX = np.concatenate([train_embeddings,test_embeddings],axis=0)\ny = np.zeros((train_embeddings.shape[0]+test_embeddings.shape[0],1))\ny[:train_embeddings.shape[0]] = 1\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"train_test_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = np.concatenate([train_embeddings,test_embeddings],axis=0)\ny = np.zeros((train_embeddings.shape[0]+test_embeddings.shape[0],1))\ny[:train_embeddings.shape[0]] = 1\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"train_test1_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model1(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.hist(oof)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Adverserial Validation: Test vs External Data (2019)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X = np.concatenate([external_train_embeddings,test_embeddings],axis=0)\ny = np.zeros((external_train_embeddings.shape[0]+test_embeddings.shape[0],1))\ny[:external_train_embeddings.shape[0]] = 1\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"ext_test_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.hist(oof)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Adverserial Validation: Test vs External Data (2017-18)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X = np.concatenate([external_train_embeddings_18,test_embeddings],axis=0)\ny = np.zeros((external_train_embeddings_18.shape[0]+test_embeddings.shape[0],1))\ny[:external_train_embeddings_18.shape[0]] = 1\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"ext_test_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Adverserial Validation: Train vs External Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X = np.concatenate([train_embeddings,external_train_embeddings],axis=0)\ny = np.zeros((train_embeddings.shape[0]+external_train_embeddings.shape[0],1))\ny[:train_embeddings.shape[0]] = 1\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"train_ext_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.hist(oof)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Adverserial Validation: Train vs External Data (2018)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X = np.concatenate([train_embeddings,external_train_embeddings_18],axis=0)\ny = np.zeros((train_embeddings.shape[0]+external_train_embeddings_18.shape[0],1))\ny[:train_embeddings.shape[0]] = 1\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"train_ext_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Adverserial Validation: External(2019) vs External(2018)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X = np.concatenate([external_train_embeddings,external_train_embeddings_18],axis=0)\ny = np.zeros((external_train_embeddings.shape[0]+external_train_embeddings_18.shape[0],1))\ny[:external_train_embeddings.shape[0]] = 1\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"train_ext_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Adverserial Validation on Non-Melanoma Samples (Train vs External 2019)\nOne of the reasons for high AUC score b/w train/test and external data may be difference in labels distribution. So, let's try to run adverserial validation on just non-melanoma samples","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X = np.concatenate([train_embeddings,external_train_embeddings],axis=0)\ny = np.zeros((train_embeddings.shape[0]+external_train_embeddings.shape[0],1))\ny[:train_embeddings.shape[0]] = 1\n\nlabels = np.concatenate([train_labels,external_train_labels],axis=0)\nX = X[labels==0]\ny = y[labels==0]\n\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"train_ext_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Adverserial Validation on Non-Melanoma Samples (Train vs External 2018)\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X = np.concatenate([train_embeddings,external_train_embeddings_18],axis=0)\ny = np.zeros((train_embeddings.shape[0]+external_train_embeddings_18.shape[0],1))\ny[:train_embeddings.shape[0]] = 1\n\nlabels = np.concatenate([train_labels,external_train_labels_18],axis=0)\nX = X[labels==0]\ny = y[labels==0]\n\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"train_ext_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Adverserial Validation on Melanoma Samples - Train vs External (2019)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X = np.concatenate([train_embeddings,external_train_embeddings],axis=0)\ny = np.zeros((train_embeddings.shape[0]+external_train_embeddings.shape[0],1))\ny[:train_embeddings.shape[0]] = 1\n\nlabels = np.concatenate([train_labels,external_train_labels],axis=0)\nX = X[labels==1]\ny = y[labels==1]\n\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"train_ext_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Adverserial Validation on Melanoma Samples - Train vs External (2018)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"X = np.concatenate([train_embeddings,external_train_embeddings_18],axis=0)\ny = np.zeros((train_embeddings.shape[0]+external_train_embeddings_18.shape[0],1))\ny[:train_embeddings.shape[0]] = 1\n\nlabels = np.concatenate([train_labels,external_train_labels_18],axis=0)\nX = X[labels==1]\ny = y[labels==1]\n\noof = np.zeros(y.shape)\nprint(X.shape,y.shape)\n\nfor i,(train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Fold:\",i,end = \" \")\n    X_train, X_test = X[train_index], X[test_index]\n    y_train, y_test = y[train_index], y[test_index]\n    model_path = \"train_ext_{}.h5\".format(i)\n    early_stop = EarlyStopping(monitor='val_auc',patience=20,verbose=1,mode='max')\n    reduce_lr = ReduceLROnPlateau(monitor='val_auc', factor=0.5, patience=4,verbose=0,mode='max')\n    checkpoint = ModelCheckpoint(model_path , monitor='val_auc', verbose=0, save_best_only=True, mode='max')\n    model = build_model(lr=0.01)\n    history = model.fit(X_train,y_train,validation_data = (X_test,y_test),verbose=0,epochs=100, batch_size = 1000,\n                        callbacks=[early_stop,reduce_lr,checkpoint])\n    model.load_weights(model_path)\n    oof[test_index] = model.predict(X_test)\n    print(\"Partial Score:\",roc_auc_score(y_test,oof[test_index]))\nprint(classification_report(y, (oof>0.5).astype(int), digits=4))\nprint(roc_auc_score(y, oof))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# It seems there is a significant difference between train and external data. What's your views??","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# External Data (2017-18) is similar to train than (2019)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Update: I wanted to try stratified-Group K Fold CV strategy for adverserial validation. But it seems last year data doesn't have patient information. \n\n# Please tell me if you come up with some improvement to this notebook. \n\n## As of now, Using 2019 data seems too risky to be used for training. Even 2018 data has AUC>0.9 for both train and test. But, I guess I will be using this because it gives a boost in both OOF and LB scores.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}