{"cells":[{"metadata":{},"cell_type":"markdown","source":"Several people have reported a discrepancy between CV and LB scores. The main idea behind this kernel is to have a quick and dirty check: how different are the distributions of the classes between training and test sets? The approach I use is adversarial validation:\n\nhttp://fastml.com/adversarial-validation-part-one/\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"from fastai import *\nfrom fastai.vision import *\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\n\nfrom sklearn.metrics import cohen_kappa_score\n\nimport numpy as np\nimport scipy as sp\nfrom functools import partial\nfrom sklearn import metrics\nfrom collections import Counter\nimport json\n\nfrom PIL import Image\n\n\nimport time\nimport torchvision\nimport torch.nn as nn\nfrom tqdm import tqdm_notebook as tqdm\n\nfrom PIL import Image, ImageFile\nfrom torch.utils.data import Dataset\nimport torch\nimport torch.optim as optim\nfrom torchvision import transforms\nfrom torch.optim import lr_scheduler\nimport os\n\ndevice = torch.device(\"cuda:0\")\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# settings\nbs = 64 \nsz = 224","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Making pretrained weights work without needing to find the default filename\nif not os.path.exists('/tmp/.cache/torch/checkpoints/'):\n        os.makedirs('/tmp/.cache/torch/checkpoints/')\n!cp '../input/resnet50/resnet50.pth' '/tmp/.cache/torch/checkpoints/resnet50-19c8e357.pth'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data"},{"metadata":{},"cell_type":"markdown","source":"The point of this block is to combine the training and test data into a single data frame, which can subsequently be used in our pipeline."},{"metadata":{"trusted":true},"cell_type":"code","source":"# training images\nbase_image_dir = os.path.join('..', 'input/aptos2019-blindness-detection/')\ntrain_dir = os.path.join(base_image_dir,'train_images/')\ndf = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))\ndf['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))\ndf = df.drop(columns=['id_code'])\ndf = df.sample(frac=1).reset_index(drop=True) #shuffle dataframe\ndf['is_test'] = 0\ndf.drop('diagnosis', axis = 1, inplace = True)\n\ndf1 = df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test images\nbase_image_dir = os.path.join('..', 'input/aptos2019-blindness-detection/')\ntrain_dir = os.path.join(base_image_dir,'test_images/')\ndf = pd.read_csv(os.path.join(base_image_dir, 'test.csv'))\ndf['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))\ndf = df.drop(columns=['id_code'])\ndf = df.sample(frac=1).reset_index(drop=True) #shuffle dataframe\ndf['is_test'] = 1\ndf2 = df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_total = pd.concat([df1,df2], axis =0 )\ndf_total = df_total.sample(frac=1).reset_index(drop=True) \ndel df1, df2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add cv folds indices (yes, i know it's ugly :-)\nkf = KFold(n_splits=5)\n\ndf_total['fold_id'] = -1\n\nfor (nf, (train_index, test_index)) in enumerate(kf.split(df_total)):\n    df_total['fold_id'][test_index] = nf\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{},"cell_type":"markdown","source":"Loop over folds - check performance for each"},{"metadata":{"trusted":true},"cell_type":"code","source":"res = np.zeros((5,1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for ii in range(0, 5):\n    \n    # create this split for training / validation \n    df = df_total.copy()\n    df['is_valid'] = (df['fold_id'] == ii) + 0\n    df.drop('fold_id', axis = 1, inplace = True)\n    \n    # create the data object\n    tfms = get_transforms(do_flip=True,flip_vert=True,max_rotate=360,max_warp=0,max_zoom=1.1,max_lighting=0.1,p_lighting=0.5)\n    src = (ImageList.from_df(df=df,path='./',cols='path') \n        .split_from_df() \n        .label_from_df(cols='is_test') \n      )\n    data= (src.transform(tfms,size=sz,resize_method=ResizeMethod.SQUISH,padding_mode='zeros')\n        .databunch(bs=bs,num_workers=4)\n        .normalize(imagenet_stats)   \n       )\n    \n    # train a model for this fold - no optimization\n    learn = cnn_learner(data, base_arch = models.resnet50)\n    learn.unfreeze()\n    learn.fit_one_cycle(1, max_lr = slice(1e-6,1e-3))\n    \n    # evaluate performance\n    img = learn.data.valid_dl\n    xpred = learn.get_preds(img)\n    xscore = roc_auc_score(xpred[1],xpred[0][:,1])\n    print('fold '+str(ii) + ': ' + str(np.round(xscore, 4)))\n\n    res[ii] = xscore\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As can be seen from the results above (each fold has AUC > 0.9), even with a clearly underfitting model (validation loss < training loss) we can quite accurately distinguish the training and test sets. This means garden variety random split just won't do the job :-("},{"metadata":{"trusted":true},"cell_type":"code","source":"print(res)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}