{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Inference kernel for [[Torch XLA] Melanoma Crazy Fast](https://www.kaggle.com/shonenkov/torch-xla-melanoma-crazy-fast)\n\nHi everyone!\n\nThis kernel is inference part for tpu training using torch/xla. \n\nSome of you have asked questions about stable roc_auc on validation and LB. It is normal, private, public and cv datasets have different distributions! Nobody can't guarantee private stage without shake up. You should understand it and use own strategy. If you think that public LB score is correct, it is your strategy. My strategy is best local cross-validation scores. In the final submission I won't choose high public score with blind blend, I will choose my best OOF prediction for my models.\n\nI wish all of us good luck on private stage!","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Main Idea\n\nUse some techniques for more stable prediction:\n\n- several checkpoints from one fold\n- ensemble of 5 folds\n- no blend of public solutions","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install -q efficientnet_pytorch > /dev/null\n!pip install --no-deps timm > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from glob import glob\nfrom tqdm import tqdm\nimport pandas as pd\nfrom sklearn.model_selection import GroupKFold\nimport cv2\nfrom skimage import io\nimport albumentations as A\nimport scipy as sp\nimport torch\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom albumentations.pytorch import ToTensor\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom torch.nn import functional as F\nfrom glob import glob\nimport sklearn\nfrom torch import nn\nimport warnings\n\nwarnings.filterwarnings(\"ignore\") \nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n\nSEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '../input/melanoma-merged-external-data-512x512-jpeg'\nTEST_ROOT_PATH = f'{DATA_PATH}/512x512-test/512x512-test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision import transforms\n\ndef get_valid_transforms():\n    return transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n    ])\n\nclass DatasetRetriever(Dataset):\n\n    def __init__(self, image_ids, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.transforms = transforms\n\n    def __getitem__(self, idx: int):\n        image_id = self.image_ids[idx]\n        image = cv2.imread(f'{TEST_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.resize(image, (256,256), cv2.INTER_AREA)\n\n        if self.transforms:\n            image = self.transforms(image)\n        return image, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(f'../input/siim-isic-melanoma-classification/test.csv', index_col='image_name')\n\ntest_dataset = DatasetRetriever(\n    image_ids=df_test.index.values,\n    transforms=get_valid_transforms(),\n)\n\ntest_loader = torch.utils.data.DataLoader(\n    test_dataset, \n    batch_size=8,\n    num_workers=2,\n    shuffle=False,\n    sampler=SequentialSampler(test_dataset),\n    pin_memory=False,\n    drop_last=False,\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Net","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import timm\n\ndef get_net():\n    net = timm.create_model('resnext50d_32x4d', pretrained=False)\n    net.fc = nn.Linear(in_features=net.fc.in_features, out_features=2, bias=True)\n    return net\n\nnet = get_net().cuda()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_inference(net, test_loader):\n    result = {'image_name': [], 'target': []}\n    for images, image_names in tqdm(test_loader, total=len(test_loader)):\n        with torch.no_grad():\n            images = images.cuda().float()\n            outputs = net(images)\n            y_pred = nn.functional.softmax(outputs, dim=1).data.cpu().numpy()[:,1]\n        result['image_name'].extend(image_names)\n        result['target'].extend(y_pred)\n    return pd.DataFrame(result).set_index('image_name')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"submissions = []\ncheckpoint_paths = sorted(glob('../input/isic-resnext50d-32x4d-public-checkpoints/*.bin'))\nfor checkpoint_path in checkpoint_paths:\n    checkpoint = torch.load(checkpoint_path)\n    net.load_state_dict(checkpoint);\n    net.eval();\n    submission = run_inference(net, test_loader)\n    submissions.append(submission)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Rankdata\nHere I would like to say very good thanks [Dmytro Danevskyi @ddanevskyi](https://www.kaggle.com/ddanevskyi) for [this topic about non-calibrated predictions](https://www.kaggle.com/c/siim-isic-melanoma-classification/discussion/156064). It really helped me, I recommend to use this technique for ensemble.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.stats import rankdata\n\nresult_submission = submissions[0].copy()\nresult_submission['target'] = 0\nfor submission in submissions:\n    result_submission['target'] += rankdata(submission['target']) / len(submissions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_submission.to_csv('submission.csv')\nresult_submission['target'].hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# OOF Evaluation","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"df_folds = pd.read_csv(f'{DATA_PATH}/folds_08062020.csv', index_col='image_id')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"TRAIN_ROOT_PATH = f'{DATA_PATH}/512x512-dataset-melanoma/512x512-dataset-melanoma'\n\ndef onehot(size, target):\n    vec = torch.zeros(size, dtype=torch.float32)\n    vec[target] = 1.\n    return vec\n\nclass DatasetRetriever(Dataset):\n\n    def __init__(self, image_ids, labels, transforms=None):\n        super().__init__()\n        self.image_ids = image_ids\n        self.labels = labels\n        self.transforms = transforms\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.resize(image, (256, 256), cv2.INTER_AREA)\n        label = self.labels[index]     \n        if self.transforms:\n            image = self.transforms(image)\n        return image, label, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def get_labels(self):\n        return list(self.labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_predictions = []\n\nfor checkpoint_paths in [\n    sorted(glob('../input/isic-resnext50d-32x4d-public-checkpoints/*-bestauc-*.bin')),\n    sorted(glob('../input/isic-resnext50d-32x4d-public-checkpoints/*-bestloss-*.bin')),\n]:\n    result = {'target': [], 'pred': [], 'image_name': []}\n    for fold_number, checkpoint_path in tqdm(enumerate(checkpoint_paths), total=len(checkpoint_paths)):\n        df_val = df_folds[(df_folds['fold'] == fold_number) & (df_folds['source'] == 'ISIC20')]\n\n        validation_dataset = DatasetRetriever(\n            image_ids=df_val.index.values,\n            labels=df_val.target.values,\n            transforms=get_valid_transforms(),\n        )\n\n        validation_loader = torch.utils.data.DataLoader(\n            validation_dataset, \n            batch_size=8,\n            num_workers=2,\n            shuffle=False,\n            sampler=SequentialSampler(validation_dataset),\n            pin_memory=False,\n            drop_last=False,\n        )\n\n        checkpoint = torch.load(checkpoint_path)\n        net.load_state_dict(checkpoint);\n        net.eval();\n\n        for step, (images, targets, image_names) in enumerate(validation_loader):\n            with torch.no_grad():\n                images = images.cuda().float()\n                outputs = net(images)\n                y_pred = nn.functional.softmax(outputs, dim=1).data.cpu().numpy()[:,1]\n            result['target'].extend(targets.numpy())\n            result['pred'].extend(y_pred)\n            result['image_name'].extend(image_names)\n            \n    oof_predictions.append(pd.DataFrame(result).set_index('image_name'))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"result_oof = oof_predictions[0].copy()\nresult_oof['pred'] = 0\nfor oof_prediction in oof_predictions:\n    result_oof['pred'] += rankdata(oof_prediction['pred']) / len(oof_predictions)\n\nprint('-'*30)\nprint(f\"[OOF RocAuc]: {sklearn.metrics.roc_auc_score(result_oof['target'], result_oof['pred']):.3f}\")\nprint(f\"[OOF AP]: {sklearn.metrics.average_precision_score(result_oof['target'], result_oof['pred']):.3f}\")\nprint('-'*30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Thank you for reading my kernel!","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}