{"cells":[{"metadata":{"_uuid":"71c8e2b868d340a3f19d473388e84db668117550"},"cell_type":"markdown","source":"This notebook tried to show that doing test time augumentation and take mean of all your predictions could boosting your score a lot. I used the same data augumentation transform in train, validation and test."},{"metadata":{"trusted":true,"_uuid":"08934db23fd405282cca0c2d1fb38397ab928e7a"},"cell_type":"code","source":"###### This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n!pip install albumentations\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n# libraries\nimport os\nimport cv2\nimport gc\nimport matplotlib.pyplot as plt\nimport random\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport time \nimport tqdm\nfrom PIL import Image\ntrain_on_gpu = True\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n\nimport albumentations\nfrom albumentations import torch as AT\n\nn_splits = 5\nbatch_size = 32\nn_epochs = 5\npatience = 15\nSEED = 323\nnum_workers = 0\nnum_tta = 64\n\ndef seed_everything(seed=SEED):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"labels = pd.read_csv('../input/histopathologic-cancer-detection/train_labels.csv')\ntr, val = train_test_split(labels.label, stratify=labels.label, test_size=0.07, random_state=SEED)\nimg_class_dict = {k:v for k, v in zip(labels.id, labels.label)}\n\nclass CancerDataset(Dataset):\n    def __init__(self, datafolder, datatype='train', idx=[], transform = transforms.Compose([transforms.CenterCrop(48),transforms.ToTensor()]), labels_dict={}):\n        self.datafolder = datafolder\n        self.datatype = datatype\n        self.image_files_list = [s for s in os.listdir(datafolder)]\n        self.image_files_list = [self.image_files_list[i] for i in idx]\n        self.transform = transform\n        self.labels_dict = labels_dict\n        if self.datatype == 'train':\n            self.labels = [labels_dict[i.split('.')[0]] for i in self.image_files_list]\n        else:\n            self.labels = [0 for _ in range(len(self.image_files_list))]\n\n    def __len__(self):\n        return len(self.image_files_list)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.datafolder, self.image_files_list[idx])\n        img = cv2.imread(img_name)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        image = self.transform(image=img)\n        image = image['image']\n\n        img_name_short = self.image_files_list[idx].split('.')[0]\n\n        if self.datatype == 'train':\n            label = self.labels_dict[img_name_short]\n        else:\n            label = 0\n        return image, label\n\n\ndata_transforms = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.RandomRotate90(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.Flip(p=0.5),\n    albumentations.OneOf([\n        albumentations.CLAHE(clip_limit=2), albumentations.IAASharpen(), albumentations.IAAEmboss(), \n        albumentations.RandomBrightness(), albumentations.RandomContrast(),\n        albumentations.JpegCompression(), albumentations.Blur(), albumentations.GaussNoise()], p=0.5), \n    albumentations.HueSaturationValue(p=0.5), \n    albumentations.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.15, rotate_limit=45, p=0.5),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\ndata_transforms_test = albumentations.Compose([\n    albumentations.Resize(224, 224),\n    albumentations.Normalize(),\n    AT.ToTensor()\n    ])\n\n\nclass Densenet169(nn.Module):\n    def __init__(self, pretrained=True):\n        super(Densenet169, self).__init__()\n        self.model = models.densenet169(pretrained=pretrained)\n        self.linear = nn.Linear(1000+2, 16)\n        self.bn = nn.BatchNorm1d(16)\n        self.dropout = nn.Dropout(0.2)\n        self.elu = nn.ELU()\n        self.out = nn.Linear(16, 1)\n    \n    def forward(self, x):\n        out = self.model(x)\n        batch = out.shape[0]\n        max_pool, _ = torch.max(out, 1, keepdim=True)\n        avg_pool = torch.mean(out, 1, keepdim=True)\n\n        out = out.view(batch, -1)\n        conc = torch.cat((out, max_pool, avg_pool), 1)\n\n        conc = self.linear(conc)\n        conc = self.elu(conc)\n        conc = self.bn(conc)\n        conc = self.dropout(conc)\n\n        res = self.out(conc)\n\n        return res\n\n\nmodel_conv = Densenet169(pretrained=False)\nmodel_conv.load_state_dict(torch.load(\"../input/densenet169-pretrain-cancer/model\"), strict=False)\nmodel_conv.eval().cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1abb2fcdc1dd65e7cf142bedf84f3749e8cc7527"},"cell_type":"code","source":"for tta in range(num_tta):\n    \n    seed_everything(SEED+10*tta)\n    test_idx = [i for i in range(len(os.listdir(\"../input/histopathologic-cancer-detection/test\")))]\n    test_set = CancerDataset(datafolder='../input/histopathologic-cancer-detection/test/', idx=test_idx, datatype='test', transform=data_transforms)\n    test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)\n    preds = []\n    for batch_i, (data, target) in enumerate(test_loader):\n        data, target = data.cuda(), target.cuda()\n        output = model_conv(data).detach()\n\n        pr = output[:,0].cpu().numpy()\n        for i in pr:\n            preds.append(i)\n        \n    test_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})\n    test_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])\n    sub = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\n    sub = pd.merge(sub, test_preds, left_on='id', right_on='imgs')\n    sub = sub[['id', 'preds']]\n    sub.columns = ['id', 'label']\n    sub.head()\n    sub.to_csv('single_model_'+str(tta)+'.csv', index=False)\n\ndel model_conv\ngc.collect()\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"942d517806f59815dd61a6af0755276132893566"},"cell_type":"code","source":"def sigmoid(x):\n  return 1 / (1 + np.exp(-x))\n\ndef sigmoid_pd(df):\n    df[\"label\"] = df[\"label\"].apply(sigmoid)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/histopathologic-cancer-detection/sample_submission.csv')\n\nfor tta in range(num_tta):\n    df0 = pd.read_csv('../working/single_model_'+str(tta)+'.csv')\n    df0 = sigmoid_pd(df0)\n    df['label'] += df0['label']\n    if(tta+1==8):\n        df_tmp = df.copy()\n        df_tmp['label'] /= 8 \n        df_tmp.to_csv('submission_tta_'+str(tta+1)+'.csv',index=False)\n    if(tta+1==16):\n        df_tmp = df.copy()\n        df_tmp['label'] /= 16\n        df_tmp.to_csv('submission_tta_'+str(tta+1)+'.csv',index=False)\n\ndf['label'] /= num_tta\ndf.to_csv('submission_tta_'+str(num_tta)+'.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}