{"cells":[{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:42:59.968582Z","iopub.status.busy":"2021-03-14T16:42:59.967997Z","iopub.status.idle":"2021-03-14T16:43:30.239975Z","shell.execute_reply":"2021-03-14T16:43:30.238958Z"},"papermill":{"duration":30.304941,"end_time":"2021-03-14T16:43:30.240195","exception":false,"start_time":"2021-03-14T16:42:59.935254","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# https://github.com/rwightman/pytorch-image-model\n!pip install ../input/timm-repo/pytorch-image-models-master/ > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:43:30.310659Z","iopub.status.busy":"2021-03-14T16:43:30.309903Z","iopub.status.idle":"2021-03-14T16:43:57.81681Z","shell.execute_reply":"2021-03-14T16:43:57.816189Z"},"papermill":{"duration":27.548212,"end_time":"2021-03-14T16:43:57.816938","exception":false,"start_time":"2021-03-14T16:43:30.268726","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# https://github.com/Cadene/pretrained-models.pytorch\n!pip install ../input/pretrainedmodels-pytorch/pretrained-models.pytorch-master/ > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:43:57.880118Z","iopub.status.busy":"2021-03-14T16:43:57.875831Z","iopub.status.idle":"2021-03-14T16:44:25.81472Z","shell.execute_reply":"2021-03-14T16:44:25.814222Z"},"papermill":{"duration":27.970102,"end_time":"2021-03-14T16:44:25.814868","exception":false,"start_time":"2021-03-14T16:43:57.844766","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# https://github.com/lukemelas/EfficientNet-PyTorch\n!pip install ../input/efficientnet-pyotrch/EfficientNet-PyTorch-master/ > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:44:25.877365Z","iopub.status.busy":"2021-03-14T16:44:25.874209Z","iopub.status.idle":"2021-03-14T16:44:53.377478Z","shell.execute_reply":"2021-03-14T16:44:53.376028Z"},"papermill":{"duration":27.535137,"end_time":"2021-03-14T16:44:53.377627","exception":false,"start_time":"2021-03-14T16:44:25.84249","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# https://github.com/qubvel/segmentation_models.pytorch\n!pip install ../input/segmentation-models-pytorch/segmentation_models.pytorch-master/ > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.026588,"end_time":"2021-03-14T16:44:53.431919","exception":false,"start_time":"2021-03-14T16:44:53.405331","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:44:53.497871Z","iopub.status.busy":"2021-03-14T16:44:53.494209Z","iopub.status.idle":"2021-03-14T16:44:54.841691Z","shell.execute_reply":"2021-03-14T16:44:54.840634Z"},"papermill":{"duration":1.382723,"end_time":"2021-03-14T16:44:54.841852","exception":false,"start_time":"2021-03-14T16:44:53.459129","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from typing import Optional, Tuple, List\nfrom os.path import join as pjoin\nfrom PIL import Image\n\nimport numpy as np\nimport pandas as pd\nimport torch\n\n\nclass RANZCRDataset(torch.utils.data.Dataset):\n def __init__(\n self,\n df,\n root,\n ext,\n path_col,\n use_timm_aug=False,\n transforms=None,\n augmentations=None,\n ):\n\n super().__init__()\n df = df.reset_index(drop=True).copy()\n self.transforms = transforms\n self.augmentations = augmentations\n self.root = root\n self.use_timm_aug = use_timm_aug\n\n self.image_names = self._prepare_image_names(df[path_col].tolist(), root, ext)\n\n def __len__(self):\n return len(self.image_names)\n\n def _prepare_image_names(self, basenames: List[str], root: str, ext: str):\n return [pjoin(root, el) + ext for el in basenames]\n\n def _prepare_img_target_from_idx(self, idx: int):\n\n image_name = self.image_names[idx]\n\n img = Image.open(image_name)\n if not self.use_timm_aug:\n img = np.array(img)\n\n if self.augmentations is not None:\n if self.use_timm_aug:\n img = self.augmentations(img)\n else:\n img = self.augmentations(image=img)[\"image\"]\n\n if self.use_timm_aug:\n img = np.array(img)\n\n if self.transforms is not None:\n img = self.transforms(img)\n\n return img\n\n def __getitem__(self, index: int):\n\n img = self._prepare_img_target_from_idx(index)\n \n return img","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.027186,"end_time":"2021-03-14T16:44:54.896281","exception":false,"start_time":"2021-03-14T16:44:54.869095","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Model"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:44:54.990545Z","iopub.status.busy":"2021-03-14T16:44:54.969582Z","iopub.status.idle":"2021-03-14T16:44:56.866173Z","shell.execute_reply":"2021-03-14T16:44:56.865251Z"},"papermill":{"duration":1.942848,"end_time":"2021-03-14T16:44:56.866315","exception":false,"start_time":"2021-03-14T16:44:54.923467","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from typing import Optional, Mapping, Any\n\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch\nimport timm\nimport segmentation_models_pytorch as smp\n\nEFFNETB6_EMB_DIM = 2304\nEFFNETB5_EMB_DIM = 2048\nEFFNETB4_EMB_DIM = 1792\nEFFNETB3_EMB_DIM = 1536\nEFFNETB1_EMB_DIM = 1280\nRESNET50_EMB_DIM = 2048\nREXNET200_EMB_DIM = 2560\nVIT_EMB_DIM = 768\nNF_RESNET50_EMB_DIM = 2048\n\nEPS = 1e-6\n\nclass TaylorSoftmax(nn.Module):\n\n def __init__(self, dim=1, n=2):\n super(TaylorSoftmax, self).__init__()\n assert n % 2 == 0\n self.dim = dim\n self.n = n\n\n def forward(self, x):\n \n fn = torch.ones_like(x)\n denor = 1.\n for i in range(1, self.n+1):\n denor *= i\n fn = fn + x.pow(i) / denor\n out = fn / fn.sum(dim=self.dim, keepdims=True)\n return out\n\n\nclass CNNModel(nn.Module):\n def __init__(\n self,\n classifiier_config: Mapping[str, Any],\n encoder_type: str,\n device: str,\n use_pretrained_encoder: bool = False,\n path_to_chkp: Optional[str] = None,\n use_taylorsoftmax: bool = False,\n one_channel: bool = True,\n ):\n super().__init__()\n\n if path_to_chkp is not None:\n use_pretrained_encoder = False\n\n if encoder_type == \"rexnet_200\":\n self.encoder = timm.create_model(\n encoder_type, pretrained=use_pretrained_encoder\n )\n if one_channel:\n self.encoder.stem.conv.in_channels = 1\n weight = self.encoder.stem.conv.weight.mean(1, keepdim=True)\n self.encoder.stem.conv.weight = torch.nn.Parameter(weight)\n self.encoder.head.fc = nn.Identity()\n nn_embed_size = REXNET200_EMB_DIM\n elif encoder_type == \"tf_efficientnet_b3_ns\":\n self.encoder = timm.create_model(\n encoder_type, pretrained=use_pretrained_encoder\n )\n if one_channel:\n self.encoder.conv_stem.in_channels = 1\n weight = self.encoder.conv_stem.weight.mean(1, keepdim=True)\n self.encoder.conv_stem.weight = torch.nn.Parameter(weight)\n self.encoder.classifier = nn.Identity()\n nn_embed_size = EFFNETB3_EMB_DIM\n elif encoder_type == \"tf_efficientnet_b5_ns\":\n self.encoder = timm.create_model(\n encoder_type, pretrained=use_pretrained_encoder, num_classes=11\n )\n if path_to_chkp is not None:\n print(\"Loading starting point\")\n state_dict = load_effnet_b5_start_point(path_to_chkp)\n self.encoder.load_state_dict(state_dict)\n if one_channel:\n self.encoder.conv_stem.in_channels = 1\n weight = self.encoder.conv_stem.weight.mean(1, keepdim=True)\n self.encoder.conv_stem.weight = torch.nn.Parameter(weight)\n self.encoder.classifier = nn.Identity()\n nn_embed_size = EFFNETB5_EMB_DIM\n elif encoder_type == \"resnet200d\":\n self.encoder = timm.create_model(\n encoder_type, pretrained=use_pretrained_encoder, num_classes=11\n )\n if path_to_chkp is not None:\n print(\"Loading starting point\")\n state_dict = load_resnet200d_start_point(path_to_chkp)\n self.encoder.load_state_dict(state_dict)\n if one_channel:\n self.encoder.conv1[0].in_channels = 1\n weight = self.encoder.conv1[0].weight.mean(1, keepdim=True)\n self.encoder.conv1[0].weight = torch.nn.Parameter(weight)\n self.encoder.fc = nn.Identity()\n nn_embed_size = RESNET50_EMB_DIM\n else:\n raise ValueError(f\"{encoder_type} is invalid model_type\")\n\n classes_num = classifiier_config[\"classes_num\"]\n hidden_dims = classifiier_config[\"hidden_dims\"]\n second_dropout_rate = classifiier_config[\"second_dropout_rate\"]\n if classifiier_config[\"classifier_type\"] == \"relu\":\n first_dropout_rate = classifiier_config[\"first_dropout_rate\"]\n self.classifier = nn.Sequential(\n nn.Linear(nn_embed_size, hidden_dims),\n nn.ReLU(),\n nn.Dropout(p=first_dropout_rate),\n nn.Linear(hidden_dims, hidden_dims),\n nn.ReLU(),\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"elu\":\n first_dropout_rate = classifiier_config[\"first_dropout_rate\"]\n self.classifier = nn.Sequential(\n nn.Dropout(first_dropout_rate),\n nn.Linear(nn_embed_size, hidden_dims),\n nn.ELU(),\n nn.Dropout(second_dropout_rate),\n nn.Linear(hidden_dims, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"dima\":\n self.classifier = nn.Sequential(\n nn.BatchNorm1d(nn_embed_size),\n nn.Linear(nn_embed_size, hidden_dims),\n nn.BatchNorm1d(hidden_dims),\n nn.PReLU(hidden_dims),\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"prelu\":\n first_dropout_rate = classifiier_config[\"first_dropout_rate\"]\n self.classifier = nn.Sequential(\n nn.Dropout(first_dropout_rate),\n nn.Linear(nn_embed_size, hidden_dims),\n nn.PReLU(hidden_dims),\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"multiscale_relu\":\n first_dropout_rate = classifiier_config[\"first_dropout_rate\"]\n self.big_dropout = nn.Dropout(p=0.5)\n self.classifier = nn.Sequential(\n nn.Linear(nn_embed_size, hidden_dims),\n nn.ELU(),\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"drop_linear\":\n self.classifier = nn.Sequential(\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(nn_embed_size, classes_num),\n )\n else:\n raise ValueError(\"Invalid classifier_type\")\n\n # Final activation\n self.use_taylorsoftmax = use_taylorsoftmax\n if self.use_taylorsoftmax:\n self.taylorsoftmax = TaylorSoftmax()\n # Classifier type\n self.classifier_type = classifiier_config[\"classifier_type\"]\n # Some additional stuff\n self.encoder_type = encoder_type\n self.device = device\n self.to(self.device)\n\n def forward(self, image):\n x = self.encoder(image)\n\n if self.classifier_type == \"multiscale_relu\":\n logits = torch.mean(\n torch.stack(\n [self.classifier(self.big_dropout(x)) for _ in range(5)],\n dim=0,\n ),\n dim=0,\n )\n else:\n logits = self.classifier(x)\n\n if self.use_taylorsoftmax:\n logits = self.taylorsoftmax(logits).log()\n\n return logits\n \n \nclass CNNSegModel(nn.Module):\n def __init__(\n self,\n classifiier_config: Mapping[str, Any],\n encoder_type: str,\n encoder_config: Mapping[str, Any],\n device: str,\n path_to_chkp: Optional[str] = None,\n use_taylorsoftmax: bool = False,\n one_channel: bool = True,\n enable_inference_mode: bool = False\n ):\n super().__init__()\n\n if path_to_chkp is not None:\n use_pretrained_encoder = False\n\n if encoder_type == \"timm-efficientnet-b5_unet\":\n self.encoder = smp.Unet(**encoder_config)\n if path_to_chkp is not None:\n print(\"Loading starting point\")\n state_dict = load_effnet_b5_start_point(path_to_chkp)\n self.encoder.encoder.load_state_dict(state_dict)\n if one_channel:\n self.encoder.encoder.conv_stem.in_channels = 1\n weight = self.encoder.encoder.conv_stem.weight.mean(1, keepdim=True)\n self.encoder.encoder.conv_stem.weight = torch.nn.Parameter(weight)\n self.encoder.classification_head[3] = nn.Identity()\n nn_embed_size = 512\n elif encoder_type == \"densenet121_unet\":\n self.encoder = smp.Unet(**encoder_config)\n if path_to_chkp is not None:\n print(\"Loading starting point\")\n state_dict = load_densenet121_start_point(path_to_chkp)\n self.encoder.encoder.load_state_dict(state_dict)\n if one_channel:\n self.encoder.encoder.features.conv0.in_channels = 1\n weight = self.encoder.encoder.features.conv0.weight.mean(\n 1, keepdim=True\n )\n self.encoder.encoder.features.conv0.weight = torch.nn.Parameter(\n weight\n )\n self.encoder.classification_head[3] = nn.Identity()\n nn_embed_size = 1024\n else:\n raise ValueError(f\"{encoder_type} is invalid model_type\")\n\n classes_num = classifiier_config[\"classes_num\"]\n hidden_dims = classifiier_config[\"hidden_dims\"]\n second_dropout_rate = classifiier_config[\"second_dropout_rate\"]\n if classifiier_config[\"classifier_type\"] == \"relu\":\n first_dropout_rate = classifiier_config[\"first_dropout_rate\"]\n self.classifier = nn.Sequential(\n nn.Linear(nn_embed_size, hidden_dims),\n nn.ReLU(),\n nn.Dropout(p=first_dropout_rate),\n nn.Linear(hidden_dims, hidden_dims),\n nn.ReLU(),\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"elu\":\n first_dropout_rate = classifiier_config[\"first_dropout_rate\"]\n self.classifier = nn.Sequential(\n nn.Dropout(first_dropout_rate),\n nn.Linear(nn_embed_size, hidden_dims),\n nn.ELU(),\n nn.Dropout(second_dropout_rate),\n nn.Linear(hidden_dims, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"dima\":\n self.classifier = nn.Sequential(\n nn.BatchNorm1d(nn_embed_size),\n nn.Linear(nn_embed_size, hidden_dims),\n nn.BatchNorm1d(hidden_dims),\n nn.PReLU(hidden_dims),\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"prelu\":\n first_dropout_rate = classifiier_config[\"first_dropout_rate\"]\n self.classifier = nn.Sequential(\n nn.Dropout(first_dropout_rate),\n nn.Linear(nn_embed_size, hidden_dims),\n nn.PReLU(hidden_dims),\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"multiscale_relu\":\n first_dropout_rate = classifiier_config[\"first_dropout_rate\"]\n self.big_dropout = nn.Dropout(p=0.5)\n self.classifier = nn.Sequential(\n nn.Linear(nn_embed_size, hidden_dims),\n nn.ELU(),\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(hidden_dims, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"drop_linear\":\n self.classifier = nn.Sequential(\n nn.Dropout(p=second_dropout_rate),\n nn.Linear(nn_embed_size, classes_num),\n )\n elif classifiier_config[\"classifier_type\"] == \"double_elu_mlp\":\n first_dropout_rate = classifiier_config[\"first_dropout_rate\"]\n self.classifier_prenet = nn.Sequential(\n nn.Dropout(first_dropout_rate),\n nn.Linear(nn_embed_size, hidden_dims),\n nn.ELU(),\n )\n self.classifier_hidden_class = nn.Sequential(\n nn.Dropout(second_dropout_rate),\n nn.Linear(\n hidden_dims, classifiier_config[\"hidden_classes_num\"]\n ),\n )\n self.classifier_class_final = nn.Sequential(\n nn.Dropout(second_dropout_rate),\n nn.Linear(\n hidden_dims + classifiier_config[\"hidden_classes_num\"],\n classes_num,\n ),\n )\n else:\n raise ValueError(\"Invalid classifier_type\")\n\n # Final activation\n self.use_taylorsoftmax = use_taylorsoftmax\n if self.use_taylorsoftmax:\n self.taylorsoftmax = TaylorSoftmax()\n # Classifier type\n self.classifier_type = classifiier_config[\"classifier_type\"]\n # Some additional stuff\n self.enable_inference_mode = enable_inference_mode\n self.encoder_type = encoder_type\n self.device = device\n self.to(self.device)\n\n def forward(self, image, enable_inference_mode=False):\n enable_inference_mode = enable_inference_mode or self.enable_inference_mode\n\n if enable_inference_mode:\n embs = self.encoder.encoder(image)\n x = self.encoder.classification_head(embs[-1])\n else:\n mask, x = self.encoder(image)\n\n if self.classifier_type == \"multiscale_relu\":\n logits = torch.mean(\n torch.stack(\n [self.classifier(self.big_dropout(x)) for _ in range(5)],\n dim=0,\n ),\n dim=0,\n )\n elif self.classifier_type == \"double_elu_mlp\":\n pre_logits = self.classifier_prenet(x)\n class_logits = self.classifier_hidden_class(pre_logits)\n logits = self.classifier_class_final(\n torch.cat([pre_logits, class_logits], axis=-1)\n )\n else:\n logits = self.classifier(x)\n\n if self.use_taylorsoftmax:\n logits = self.taylorsoftmax(logits).log()\n\n if enable_inference_mode:\n return logits\n else:\n if self.classifier_type == \"double_elu_mlp\":\n return mask, class_logits, logits\n else:\n return mask, logits","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.026987,"end_time":"2021-03-14T16:44:56.92047","exception":false,"start_time":"2021-03-14T16:44:56.893483","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Inference utils"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:44:56.987983Z","iopub.status.busy":"2021-03-14T16:44:56.987144Z","iopub.status.idle":"2021-03-14T16:44:56.991Z","shell.execute_reply":"2021-03-14T16:44:56.990581Z"},"papermill":{"duration":0.043297,"end_time":"2021-03-14T16:44:56.991116","exception":false,"start_time":"2021-03-14T16:44:56.947819","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import os\n\nfrom typing import List, Optional, Callable, Mapping, Any\nfrom collections import OrderedDict\nfrom tqdm import tqdm\n\nimport numpy as np\nimport torch\n\ndef get_validation_models(\n model_initilizer: Callable,\n model_config: Mapping[str, Any],\n model_ckp_dicts: List[OrderedDict],\n device: str,\n):\n t_models = []\n\n for mcd in model_ckp_dicts:\n\n t_model = model_initilizer(**model_config, device=device)\n t_model.load_state_dict(mcd)\n t_model = t_model.to(device)\n t_model.eval()\n t_models.append(t_model)\n\n return t_models\n\n\ndef create_val_loaders(\n loader_initilizer: object,\n loader_config: Mapping[str, Any],\n dfs: List[str],\n batch_size: int,\n):\n t_loaders = []\n\n for df in dfs:\n t_dataset = loader_initilizer(df=df, **loader_config)\n t_loader = torch.utils.data.DataLoader(\n t_dataset,\n batch_size=batch_size,\n drop_last=False,\n shuffle=False,\n num_workers=os.cpu_count() // 2,\n )\n\n t_loaders.append(t_loader)\n\n return t_loaders\n\n\n@torch.no_grad()\ndef cnn_model_predict(t_batch, t_model, t_device):\n image = t_batch.to(t_device)\n logits = t_model(image)\n logits = logits.detach().cpu().numpy()\n return logits\n\n\ndef predict_over_all_train(\n my_loaders, my_models, model_predict_func, device, do_concat=True\n):\n logits = []\n for loader, model in zip(my_loaders, my_models):\n for batch in tqdm(loader):\n logit = model_predict_func(batch, model, device)\n logits.append(logit)\n\n if do_concat:\n logits = np.concatenate(logits)\n\n return logits\n\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.027351,"end_time":"2021-03-14T16:44:57.0458","exception":false,"start_time":"2021-03-14T16:44:57.018449","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Main imports"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:44:57.106666Z","iopub.status.busy":"2021-03-14T16:44:57.106013Z","iopub.status.idle":"2021-03-14T16:44:58.967206Z","shell.execute_reply":"2021-03-14T16:44:58.96617Z"},"papermill":{"duration":1.894629,"end_time":"2021-03-14T16:44:58.967343","exception":false,"start_time":"2021-03-14T16:44:57.072714","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import os\n\nfrom glob import glob\nfrom os.path import splitext\n\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nimport cv2\nimport torch.nn as nn\nimport numpy as np\nimport pandas as pd\nimport torchvision.transforms as T\nimport albumentations as albu\n\nfrom scipy.special import softmax\nfrom matplotlib import pyplot as plt\nfrom torch.utils.data import DataLoader, Dataset\nfrom albumentations import Compose, Resize, Normalize\nfrom albumentations.pytorch import ToTensorV2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:44:59.026453Z","iopub.status.busy":"2021-03-14T16:44:59.025715Z","iopub.status.idle":"2021-03-14T16:44:59.028703Z","shell.execute_reply":"2021-03-14T16:44:59.028249Z"},"papermill":{"duration":0.03409,"end_time":"2021-03-14T16:44:59.028812","exception":false,"start_time":"2021-03-14T16:44:58.994722","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"SKIP_VAL = True","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Public models"},{"metadata":{"trusted":true},"cell_type":"code","source":"def public_notebook():\n device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n \n BATCH_SIZE = 64\n TEST_PATH = '../input/ranzcr-clip-catheter-line-classification/test'\n \n test = pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv')\n \n class TestDataset(Dataset):\n def __init__(self, df, transform=None):\n self.df = df\n self.file_names = df['StudyInstanceUID'].values\n self.transform = transform\n\n def __len__(self):\n return len(self.df)\n\n def __getitem__(self, idx):\n file_name = self.file_names[idx]\n file_path = f'{TEST_PATH}/{file_name}.jpg'\n image = cv2.imread(file_path)\n image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n if self.transform:\n augmented = self.transform(image=image)\n image = augmented['image']\n return image\n \n def get_transforms(image_size=640):\n return Compose([\n Resize(image_size, image_size),\n Normalize(),\n ToTensorV2(),\n ])\n \n class ResNet200D(nn.Module):\n def __init__(self, model_name='resnet200d'):\n super().__init__()\n self.model = timm.create_model(model_name, pretrained=False)\n n_features = self.model.fc.in_features\n self.model.global_pool = nn.Identity()\n self.model.fc = nn.Identity()\n self.pooling = nn.AdaptiveAvgPool2d(1)\n self.fc = nn.Linear(n_features, 11)\n\n def forward(self, x):\n bs = x.size(0)\n features = self.model(x)\n pooled_features = self.pooling(features).view(bs, -1)\n output = self.fc(pooled_features)\n return output\n\n class SeResNet152D(nn.Module):\n def __init__(self, model_name='seresnet152d'):\n super().__init__()\n self.model = timm.create_model(model_name, pretrained=False)\n n_features = self.model.fc.in_features\n self.model.global_pool = nn.Identity()\n self.model.fc = nn.Identity()\n self.pooling = nn.AdaptiveAvgPool2d(1)\n self.fc = nn.Linear(n_features, 11)\n\n def forward(self, x):\n bs = x.size(0)\n features = self.model(x)\n pooled_features = self.pooling(features).view(bs, -1)\n output = self.fc(pooled_features)\n return output\n\n class RANZCRResNet200D(nn.Module):\n def __init__(self, model_name='resnet200d', out_dim=11, pretrained=False):\n super().__init__()\n self.model = timm.create_model(model_name, pretrained=False)\n n_features = self.model.fc.in_features\n self.model.global_pool = nn.Identity()\n self.model.fc = nn.Identity()\n self.pooling = nn.AdaptiveAvgPool2d(1)\n self.fc = nn.Linear(n_features, out_dim)\n\n def forward(self, x):\n bs = x.size(0)\n features = self.model(x)\n pooled_features = self.pooling(features).view(bs, -1)\n output = self.fc(pooled_features)\n return output\n \n def inference(models, test_loader, device):\n tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n probs = []\n for i, (images) in tk0:\n images = images.to(device)\n avg_preds = []\n for model in models:\n with torch.no_grad():\n y_preds1 = model(images)\n y_preds2 = model(images.flip(-1))\n y_preds = (\n y_preds1.sigmoid().to('cpu').numpy() + \n y_preds2.sigmoid().to('cpu').numpy()\n ) / 2\n avg_preds.append(y_preds)\n avg_preds = np.stack(avg_preds, axis=0)\n probs.append(avg_preds)\n probs = np.concatenate(probs, axis=1)\n return probs\n \n models200D = []\n model = ResNet200D()\n model.load_state_dict(torch.load(\"../input/resnet200d-public/resnet200d_320_CV9632.pth\")['model'])\n model.eval()\n model.to(device)\n models200D.append(model)\n\n models200D_2 = []\n model = RANZCRResNet200D()\n model.load_state_dict(torch.load(\"../input/resnet200d-baseline-benchmark-public/resnet200d_fold0_cv953.pth\", map_location='cuda:0'))\n model.eval()\n model.to(device)\n models200D_2.append(model)\n\n model = RANZCRResNet200D()\n model.load_state_dict(torch.load(\"../input/resnet200d-baseline-benchmark-public/resnet200d_fold1_cv955.pth\", map_location='cuda:0'))\n model.eval()\n model.to(device)\n models200D_2.append(model)\n\n model = RANZCRResNet200D()\n model.load_state_dict(torch.load(\"../input/resnet200d-baseline-benchmark-public/resnet200d_fold2_cv955.pth\", map_location='cuda:0'))\n model.eval()\n model.to(device)\n models200D_2.append(model)\n\n model = RANZCRResNet200D()\n model.load_state_dict(torch.load(\"../input/resnet200d-baseline-benchmark-public/resnet200d_fold3_cv957.pth\", map_location='cuda:0'))\n model.eval()\n model.to(device)\n models200D_2.append(model)\n\n model = RANZCRResNet200D()\n model.load_state_dict(torch.load(\"../input/resnet200d-baseline-benchmark-public/resnet200d_fold4_cv954.pth\", map_location='cuda:0'))\n model.eval()\n model.to(device)\n models200D_2.append(model)\n\n model = SeResNet152D()\n model.load_state_dict(torch.load('../input/seresnet152d-cv9615/seresnet152d_320_CV96.15.pth')['model'])\n model.eval()\n model.to(device)\n models200D.append(model)\n \n test_dataset_512 = TestDataset(test, transform=get_transforms(image_size=512))\n test_loader_512 = DataLoader(test_dataset_512, batch_size=BATCH_SIZE, shuffle=False, num_workers=os.cpu_count() // 2 , pin_memory=True)\n\n test_dataset_640 = TestDataset(test, transform=get_transforms(image_size=640))\n test_loader_640 = DataLoader(test_dataset_640, batch_size=BATCH_SIZE, shuffle=False, num_workers=os.cpu_count() // 2 , pin_memory=True)\n\n predictions200d = inference(models200D, test_loader_640, device)\n predictions200d_2 = inference(models200D_2, test_loader_512, device)\n \n return predictions200d, predictions200d_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pred_1, pred_2 = public_notebook()\n\n\n# public_pred_1, public_pred_3 = pred_1[0], pred_1[1]\n# public_pred_2 = pred_2.mean(0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# '../input/ranzcr-subs/resnet200d-inference-single-model-lb-96-5.csv' - public_pred_1\n# '../input/ranzcr-subs/resnet200d-public-benchmark-2xtta-lb0-965.csv' - public_pred_2\n# '../input/ranzcr-subs/ranzcr-resnet200d-seresnet152d-inference.csv' - public_pred_3\n# '../input/ranzcr-subs/resnet-c12.csv' - public_pred_2","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.026958,"end_time":"2021-03-14T16:44:59.082759","exception":false,"start_time":"2021-03-14T16:44:59.055801","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Resize images"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:44:59.142417Z","iopub.status.busy":"2021-03-14T16:44:59.14165Z","iopub.status.idle":"2021-03-14T16:44:59.144679Z","shell.execute_reply":"2021-03-14T16:44:59.144266Z"},"papermill":{"duration":0.035076,"end_time":"2021-03-14T16:44:59.144791","exception":false,"start_time":"2021-03-14T16:44:59.109715","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def create_folder(folder_name):\n if os.path.exists(folder_name):\n !rm -rf {folder_name}\n \n os.makedirs(folder_name)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:44:59.202081Z","iopub.status.busy":"2021-03-14T16:44:59.201446Z","iopub.status.idle":"2021-03-14T16:44:59.204235Z","shell.execute_reply":"2021-03-14T16:44:59.203844Z"},"papermill":{"duration":0.03284,"end_time":"2021-03-14T16:44:59.204341","exception":false,"start_time":"2021-03-14T16:44:59.171501","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"RESIZE_SIZE = 640","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:44:59.26505Z","iopub.status.busy":"2021-03-14T16:44:59.264559Z","iopub.status.idle":"2021-03-14T16:50:23.42076Z","shell.execute_reply":"2021-03-14T16:50:23.421186Z"},"papermill":{"duration":324.190398,"end_time":"2021-03-14T16:50:23.421352","exception":false,"start_time":"2021-03-14T16:44:59.230954","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test_im_names = glob('../input/ranzcr-clip-catheter-line-classification/test/*.jpg')\n\n\ncreate_folder('test_images_512_512')\n\nfor im_n in tqdm(test_im_names):\n im = Image.open(im_n)\n im = im.resize((RESIZE_SIZE, RESIZE_SIZE), Image.ANTIALIAS)\n name, _ = splitext(os.path.basename(im_n))\n im_name = name + '.jpeg'\n im.save(pjoin('test_images_512_512', im_name), 'JPEG', quality=100)\n \n\nif not SKIP_VAL:\n train_im_names = glob('../input/ranzcr-clip-catheter-line-classification/train/*.jpg')\n\n create_folder('train_images_512_512')\n\n for im_n in tqdm(train_im_names):\n im = Image.open(im_n)\n im = im.resize((RESIZE_SIZE, RESIZE_SIZE), Image.ANTIALIAS)\n name, _ = splitext(os.path.basename(im_n))\n im_name = name + '.jpeg'\n im.save(pjoin('train_images_512_512', im_name),'JPEG', quality=100)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.520454,"end_time":"2021-03-14T16:50:24.464514","exception":false,"start_time":"2021-03-14T16:50:23.94406","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Read data"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:50:25.526728Z","iopub.status.busy":"2021-03-14T16:50:25.526148Z","iopub.status.idle":"2021-03-14T16:50:25.654089Z","shell.execute_reply":"2021-03-14T16:50:25.652749Z"},"papermill":{"duration":0.673621,"end_time":"2021-03-14T16:50:25.654223","exception":false,"start_time":"2021-03-14T16:50:24.980602","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"PATH2DIR = '../input/ranzcr-clip-catheter-line-classification/'\nos.listdir(PATH2DIR)\n\ntrain = pd.read_csv(pjoin(PATH2DIR, 'train.csv'))\nsample_sub = pd.read_csv(pjoin(PATH2DIR, 'sample_submission.csv'))\nsplit = np.load('../input/ranzcr-models/naive_cv_split.npy', allow_pickle=True)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.645175,"end_time":"2021-03-14T16:50:27.13531","exception":false,"start_time":"2021-03-14T16:50:26.490135","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Model"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:50:28.150658Z","iopub.status.busy":"2021-03-14T16:50:28.14899Z","iopub.status.idle":"2021-03-14T16:50:28.15126Z","shell.execute_reply":"2021-03-14T16:50:28.151675Z"},"papermill":{"duration":0.50511,"end_time":"2021-03-14T16:50:28.151805","exception":false,"start_time":"2021-03-14T16:50:27.646695","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"DEVICE = 'cuda'","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:50:29.149832Z","iopub.status.busy":"2021-03-14T16:50:29.149237Z","iopub.status.idle":"2021-03-14T16:50:29.159939Z","shell.execute_reply":"2021-03-14T16:50:29.160342Z"},"papermill":{"duration":0.513668,"end_time":"2021-03-14T16:50:29.160491","exception":false,"start_time":"2021-03-14T16:50:28.646823","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"os.listdir('../input/ranzcr-models/')","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.503759,"end_time":"2021-03-14T16:50:30.162833","exception":false,"start_time":"2021-03-14T16:50:29.659074","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 512 Res"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:50:31.166978Z","iopub.status.busy":"2021-03-14T16:50:31.165318Z","iopub.status.idle":"2021-03-14T16:50:31.170252Z","shell.execute_reply":"2021-03-14T16:50:31.169844Z"},"papermill":{"duration":0.508791,"end_time":"2021-03-14T16:50:31.170363","exception":false,"start_time":"2021-03-14T16:50:30.661572","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"[el for el in os.listdir('../input/ranzcr-models/') if '512res' in el]","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:50:32.522675Z","iopub.status.busy":"2021-03-14T16:50:32.52135Z","iopub.status.idle":"2021-03-14T16:50:50.012601Z","shell.execute_reply":"2021-03-14T16:50:50.011562Z"},"papermill":{"duration":18.341154,"end_time":"2021-03-14T16:50:50.012761","exception":false,"start_time":"2021-03-14T16:50:31.671607","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"models_512 = []\n\n# Public 0.\nckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_32bs_640res_lesslaugs_ls005_shedchanged_startpoint_difflrs_segbranch_125coefs_1e4noseg_bigholes_firstpseudo_swa_roc_auc_score/timm_efficientnet_b5_unet_32bs_640res_lesslaugs_ls005_shedchanged_startpoint_difflrs_segbranch_125coefs_1e4noseg_bigholes_firstpseudo_swa_roc_auc_score/*.pt')\nprint(ckp_names)\nchckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\nmodels_512.append(get_validation_models(\n model_initilizer=CNNSegModel,\n model_config={\n \"classifiier_config\": {\n \"classifier_type\": \"elu\",\n \"classes_num\": 11,\n \"hidden_dims\": 1024,\n \"second_dropout_rate\": 0.2,\n \"first_dropout_rate\": 0.3,\n },\n \"encoder_config\":{\n \"in_channels\":3,\n \"encoder_name\":'timm-efficientnet-b5', \n \"encoder_weights\":None, \n \"classes\":2, \n \"activation\":'sigmoid',\n \"aux_params\":dict(\n pooling='avg', # one of 'avg', 'max'\n dropout=None, # dropout ratio, default is None\n classes=4, # define number of output labels\n )\n },\n \"encoder_type\": \"timm-efficientnet-b5_unet\",\n \"use_taylorsoftmax\": False,\n \"one_channel\": True,\n \"enable_inference_mode\": True\n },\n model_ckp_dicts=chckps,\n device=DEVICE\n))\n\n# # Public 0.\n# ckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_selected2steppseudo_swa_roc_auc_score/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_selected2steppseudo_swa_roc_auc_score/*.pt')\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\":{\n# \"in_channels\":3,\n# \"encoder_name\":'timm-efficientnet-b5', \n# \"encoder_weights\":None, \n# \"classes\":2, \n# \"activation\":'sigmoid',\n# \"aux_params\":dict(\n# pooling='avg', # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# )\n# },\n# \"encoder_type\": \"timm-efficientnet-b5_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.963\n# ckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_32bs_640res_lesslaugs_ls005_shedchanged_startpoint_difflrs_segbranch_125coefs_1e4noseg_bigholes_best_roc_auc_score/timm_efficientnet_b5_unet_32bs_640res_lesslaugs_ls005_shedchanged_startpoint_difflrs_segbranch_125coefs_1e4noseg_bigholes_best_roc_auc_score/*.pt')\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\":{\n# \"in_channels\":3,\n# \"encoder_name\":'timm-efficientnet-b5', \n# \"encoder_weights\":None, \n# \"classes\":2, \n# \"activation\":'sigmoid',\n# \"aux_params\":dict(\n# pooling='avg', # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# )\n# },\n# \"encoder_type\": \"timm-efficientnet-b5_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.963\n# ckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_32bs_640res_lesslaugs_ls005_shedchanged_startpoint_difflrs_segbranch_125coefs_1e4noseg_bigholes_swa_roc_auc_score/timm_efficientnet_b5_unet_32bs_640res_lesslaugs_ls005_shedchanged_startpoint_difflrs_segbranch_125coefs_1e4noseg_bigholes_swa_roc_auc_score/*.pt')\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\":{\n# \"in_channels\":3,\n# \"encoder_name\":'timm-efficientnet-b5', \n# \"encoder_weights\":None, \n# \"classes\":2, \n# \"activation\":'sigmoid',\n# \"aux_params\":dict(\n# pooling='avg', # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# )\n# },\n# \"encoder_type\": \"timm-efficientnet-b5_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# Public 0.961\nckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_32bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_morepseudo_swa_roc_auc_score/timm_efficientnet_b5_unet_32bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_morepseudo_swa_roc_auc_score/*.pt')\nprint(ckp_names)\nchckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\nmodels_512.append(get_validation_models(\n model_initilizer=CNNSegModel,\n model_config={\n \"classifiier_config\": {\n \"classifier_type\": \"elu\",\n \"classes_num\": 11,\n \"hidden_dims\": 1024,\n \"second_dropout_rate\": 0.2,\n \"first_dropout_rate\": 0.3,\n },\n \"encoder_config\":{\n \"in_channels\":3,\n \"encoder_name\":'timm-efficientnet-b5', \n \"encoder_weights\":None, \n \"classes\":2, \n \"activation\":'sigmoid',\n \"aux_params\":dict(\n pooling='avg', # one of 'avg', 'max'\n dropout=None, # dropout ratio, default is None\n classes=4, # define number of output labels\n )\n },\n \"encoder_type\": \"timm-efficientnet-b5_unet\",\n \"use_taylorsoftmax\": False,\n \"one_channel\": True,\n \"enable_inference_mode\": True\n },\n model_ckp_dicts=chckps,\n device=DEVICE\n))\n\n# # Public 0.959\n# ckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_pseudo_best_roc_auc_score/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_pseudo_best_roc_auc_score/*.pt')\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\":{\n# \"in_channels\":3,\n# \"encoder_name\":'timm-efficientnet-b5', \n# \"encoder_weights\":None, \n# \"classes\":2, \n# \"activation\":'sigmoid',\n# \"aux_params\":dict(\n# pooling='avg', # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# )\n# },\n# \"encoder_type\": \"timm-efficientnet-b5_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# Public 0.961\nckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_pseudo_swa_roc_auc_score/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_pseudo_swa_roc_auc_score/*.pt')\nprint(ckp_names)\nchckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\nmodels_512.append(get_validation_models(\n model_initilizer=CNNSegModel,\n model_config={\n \"classifiier_config\": {\n \"classifier_type\": \"elu\",\n \"classes_num\": 11,\n \"hidden_dims\": 1024,\n \"second_dropout_rate\": 0.2,\n \"first_dropout_rate\": 0.3,\n },\n \"encoder_config\":{\n \"in_channels\":3,\n \"encoder_name\":'timm-efficientnet-b5', \n \"encoder_weights\":None, \n \"classes\":2, \n \"activation\":'sigmoid',\n \"aux_params\":dict(\n pooling='avg', # one of 'avg', 'max'\n dropout=None, # dropout ratio, default is None\n classes=4, # define number of output labels\n )\n },\n \"encoder_type\": \"timm-efficientnet-b5_unet\",\n \"use_taylorsoftmax\": False,\n \"one_channel\": True,\n \"enable_inference_mode\": True\n },\n model_ckp_dicts=chckps,\n device=DEVICE\n))\n\n# # Public 0.\n# ckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_rotaugs_ls005_shedchanged_startpoint_difflrs_segbranch_124coefs_1e4noseg_doublemlp_best_roc_auc_score/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_rotaugs_ls005_shedchanged_startpoint_difflrs_segbranch_124coefs_1e4noseg_doublemlp_best_roc_auc_score/*.pt')\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"double_elu_mlp\",\n# \"classes_num\": 11,\n# \"hidden_classes_num\": 4,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\": {\n# \"in_channels\": 3,\n# \"encoder_name\": \"timm-efficientnet-b5\",\n# \"encoder_weights\": None,\n# \"classes\": 12,\n# \"activation\": \"sigmoid\",\n# \"aux_params\": dict(\n# pooling=\"avg\", # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# ),\n# },\n# \"encoder_type\": \"timm-efficientnet-b5_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# #\"path_to_chkp\": \"/data/additional_data/startingpoints/densenet121_chestx.pth\",\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.\n# ckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_rotaugs_ls005_shedchanged_startpoint_difflrs_segbranch_124coefs_1e4noseg_doublemlp_swa_roc_auc_score/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_rotaugs_ls005_shedchanged_startpoint_difflrs_segbranch_124coefs_1e4noseg_doublemlp_swa_roc_auc_score/*.pt')\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"double_elu_mlp\",\n# \"classes_num\": 11,\n# \"hidden_classes_num\": 4,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\": {\n# \"in_channels\": 3,\n# \"encoder_name\": \"timm-efficientnet-b5\",\n# \"encoder_weights\": None,\n# \"classes\": 12,\n# \"activation\": \"sigmoid\",\n# \"aux_params\": dict(\n# pooling=\"avg\", # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# ),\n# },\n# \"encoder_type\": \"timm-efficientnet-b5_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# #\"path_to_chkp\": \"/data/additional_data/startingpoints/densenet121_chestx.pth\",\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.93\n# ckp_names = glob('../input/ranzcr-models/densenet121_unet_52bs_640res_qubvelaugs_rotaugs_ls005_shedchanged_startpoint_difflrs_segbranch_124coefs_1e4noseg_best_roc_auc_score/densenet121_unet_52bs_640res_qubvelaugs_rotaugs_ls005_shedchanged_startpoint_difflrs_segbranch_124coefs_1e4noseg_best_roc_auc_score/*.pt')\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# #\"hidden_classes_num\": 4,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\": {\n# \"in_channels\": 3,\n# \"encoder_name\": \"densenet121\",\n# \"encoder_weights\": None,\n# \"classes\": 2,\n# \"activation\": \"sigmoid\",\n# \"aux_params\": dict(\n# pooling=\"avg\", # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# ),\n# },\n# \"encoder_type\": \"densenet121_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# #\"path_to_chkp\": \"/data/additional_data/startingpoints/densenet121_chestx.pth\",\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.953\n# ckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_rotaugs_ls005_shedchanged_startpoint_difflrs_segbranch_124coefs_1e4noseg_multichanneltube_mtm_swa_roc_auc_score/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_rotaugs_ls005_shedchanged_startpoint_difflrs_segbranch_124coefs_1e4noseg_multichanneltube_mtm_swa_roc_auc_score/*.pt')\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\":{\n# \"in_channels\":3,\n# \"encoder_name\":'timm-efficientnet-b5', \n# \"encoder_weights\":None, \n# \"classes\":12, \n# \"activation\":'sigmoid',\n# \"aux_params\":dict(\n# pooling='avg', # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# )\n# },\n# \"encoder_type\": \"timm-efficientnet-b5_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# Public 0.965\nckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_swa_roc_auc_score/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_swa_roc_auc_score/*.pt')\nprint(ckp_names)\nchckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\nmodels_512.append(get_validation_models(\n model_initilizer=CNNSegModel,\n model_config={\n \"classifiier_config\": {\n \"classifier_type\": \"elu\",\n \"classes_num\": 11,\n \"hidden_dims\": 1024,\n \"second_dropout_rate\": 0.2,\n \"first_dropout_rate\": 0.3,\n },\n \"encoder_config\":{\n \"in_channels\":3,\n \"encoder_name\":'timm-efficientnet-b5', \n \"encoder_weights\":None, \n \"classes\":2, \n \"activation\":'sigmoid',\n \"aux_params\":dict(\n pooling='avg', # one of 'avg', 'max'\n dropout=None, # dropout ratio, default is None\n classes=4, # define number of output labels\n )\n },\n \"encoder_type\": \"timm-efficientnet-b5_unet\",\n \"use_taylorsoftmax\": False,\n \"one_channel\": True,\n \"enable_inference_mode\": True\n },\n model_ckp_dicts=chckps,\n device=DEVICE\n))\n\n# # Public 0.963\n# ckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_best_roc_auc_score/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_1e4noseg_bigholes_best_roc_auc_score/*.pt')\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\":{\n# \"in_channels\":3,\n# \"encoder_name\":'timm-efficientnet-b5', \n# \"encoder_weights\":None, \n# \"classes\":2, \n# \"activation\":'sigmoid',\n# \"aux_params\":dict(\n# pooling='avg', # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# )\n# },\n# \"encoder_type\": \"timm-efficientnet-b5_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n\n# # Public 0.962\n# ckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_best_roc_auc_score/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_best_roc_auc_score/*.pt')\n# ckp_names = sorted(ckp_names, key=lambda x: int(x.split('_')[-1].split('.')[0]))\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\":{\n# \"in_channels\":3,\n# \"encoder_name\":'timm-efficientnet-b5', \n# \"encoder_weights\":None, \n# \"classes\":2, \n# \"activation\":'sigmoid',\n# \"aux_params\":dict(\n# pooling='avg', # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# )\n# },\n# \"encoder_type\": \"timm-efficientnet-b5_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.96\n# ckp_names = glob('../input/ranzcr-models/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_swa_roc_auc_score/timm_efficientnet_b5_unet_24bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_segbranch_113coefs_swa_roc_auc_score/*.pt')\n# ckp_names = sorted(ckp_names, key=lambda x: int(x.split('_')[-1].split('.')[0]))\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNSegModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_config\":{\n# \"in_channels\":3,\n# \"encoder_name\":'timm-efficientnet-b5', \n# \"encoder_weights\":None, \n# \"classes\":2, \n# \"activation\":'sigmoid',\n# \"aux_params\":dict(\n# pooling='avg', # one of 'avg', 'max'\n# dropout=None, # dropout ratio, default is None\n# classes=4, # define number of output labels\n# )\n# },\n# \"encoder_type\": \"timm-efficientnet-b5_unet\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# \"enable_inference_mode\": True\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.951\n# ckp_names = glob('../input/ranzcr-models/tf_efficientnet_b3_ns_20bs_512res_specialaugs_ls005_shedchanged_swa_roc_auc_score/tf_efficientnet_b3_ns_20bs_512res_specialaugs_ls005_shedchanged_swa_roc_auc_score/*.pt')\n# ckp_names = sorted(ckp_names, key=lambda x: int(x.split('_')[-1].split('.')[0]))\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_type\": \"tf_efficientnet_b3_ns\",\n# \"use_taylorsoftmax\": False\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.955\n# ckp_names = glob('../input/ranzcr-models/rexnet_200_18bs_512res_specialaugs_ls005_shedchanged_swa_roc_auc_score/rexnet_200_18bs_512res_specialaugs_ls005_shedchanged_swa_roc_auc_score/*.pt')\n# ckp_names = sorted(ckp_names, key=lambda x: int(x.split('_')[-1].split('.')[0]))\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_type\": \"rexnet_200\",\n# \"use_taylorsoftmax\": False\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.95\n# ckp_names = glob('../input/ranzcr-models/tf_efficientnet_b5_ns_32bs_640res_qubvelaugs_ls005_shedchanged_smallerler_swa_roc_auc_score/tf_efficientnet_b5_ns_32bs_640res_qubvelaugs_ls005_shedchanged_smallerler_swa_roc_auc_score/*.pt')\n# ckp_names = sorted(ckp_names, key=lambda x: int(x.split('_')[-1].split('.')[0]))\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_type\": \"tf_efficientnet_b5_ns\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.958\n# ckp_names = glob('../input/ranzcr-models/tf_efficientnet_b5_ns_32bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_swa_roc_auc_score/tf_efficientnet_b5_ns_32bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_swa_roc_auc_score/*.pt')\n# ckp_names = sorted(ckp_names, key=lambda x: int(x.split('_')[-1].split('.')[0]))\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_type\": \"tf_efficientnet_b5_ns\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.955\n# ckp_names = glob('../input/ranzcr-models/rexnet_200_32bs_512res_fmix_specialaugs_ls005_shedchanged_swa_roc_auc_score/rexnet_200_32bs_512res_fmix_specialaugs_ls005_shedchanged_swa_roc_auc_score/*.pt')\n# ckp_names = sorted(ckp_names, key=lambda x: int(x.split('_')[-1].split('.')[0]))\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_type\": \"rexnet_200\",\n# \"use_taylorsoftmax\": False\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.957\n# ckp_names = glob('../input/ranzcr-models/resnet200d_40bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_swa_roc_auc_score/resnet200d_40bs_640res_qubvelaugs_ls005_shedchanged_startpoint_difflrs_swa_roc_auc_score/*.pt')\n# ckp_names = sorted(ckp_names, key=lambda x: int(x.split('_')[-1].split('.')[0]))\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_type\": \"resnet200d\",\n# \"use_taylorsoftmax\": False,\n# \"one_channel\": True,\n# #\"path_to_chkp\": \"/data/additional_data/startingpoints/resnet200d_320_chestx.pth\"\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))\n\n# # Public 0.955\n# ckp_names = glob('../input/ranzcr-models/rexnet_200_32bs_512res_cutmix_qubvelaug_noblur_ls005_nocutout_shedchanged_swa_roc_auc_score/rexnet_200_32bs_512res_cutmix_qubvelaug_noblur_ls005_nocutout_shedchanged_swa_roc_auc_score/*.pt')\n# ckp_names = sorted(ckp_names, key=lambda x: int(x.split('_')[-1].split('.')[0]))\n# print(ckp_names)\n# chckps = [torch.load(el, map_location='cpu') for el in ckp_names]\n\n# models_512.append(get_validation_models(\n# model_initilizer=CNNModel,\n# model_config={\n# \"classifiier_config\": {\n# \"classifier_type\": \"elu\",\n# \"classes_num\": 11,\n# \"hidden_dims\": 1024,\n# \"second_dropout_rate\": 0.2,\n# \"first_dropout_rate\": 0.3,\n# },\n# \"encoder_type\": \"rexnet_200\",\n# \"use_taylorsoftmax\": False\n# },\n# model_ckp_dicts=chckps,\n# device=DEVICE\n# ))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.513782,"end_time":"2021-03-14T16:50:51.035442","exception":false,"start_time":"2021-03-14T16:50:50.52166","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Val Predict"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:50:52.089407Z","iopub.status.busy":"2021-03-14T16:50:52.087623Z","iopub.status.idle":"2021-03-14T16:50:52.089972Z","shell.execute_reply":"2021-03-14T16:50:52.090358Z"},"papermill":{"duration":0.510455,"end_time":"2021-03-14T16:50:52.090506","exception":false,"start_time":"2021-03-14T16:50:51.580051","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"if not SKIP_VAL:\n\n val_dfs = [\n train.iloc[split[i][1]] for i in range(5)\n ]\n\n val_loaders = create_val_loaders(\n loader_initilizer=RANZCRDataset,\n loader_config={\n \"root\":'train_images_512_512',\n \"path_col\": \"StudyInstanceUID\",\n \"ext\": \".jpeg\",\n \"transforms\":T.ToTensor()\n },\n dfs=val_dfs,\n batch_size=32\n )\n \n train_logits = predict_over_all_train(\n val_loaders,\n models_512[0],\n cnn_model_predict,\n DEVICE\n )\n \n val_dfs = pd.concat(val_dfs).reset_index(drop=True)\n\n oof_score = roc_auc_score(val_dfs.iloc[:,1:-1], train_logits)\n\n print(f\"OOF score : {oof_score}\")","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.497697,"end_time":"2021-03-14T16:50:53.08586","exception":false,"start_time":"2021-03-14T16:50:52.588163","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Predict Test"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:50:54.145752Z","iopub.status.busy":"2021-03-14T16:50:54.144013Z","iopub.status.idle":"2021-03-14T16:50:54.146334Z","shell.execute_reply":"2021-03-14T16:50:54.146739Z"},"papermill":{"duration":0.563827,"end_time":"2021-03-14T16:50:54.146862","exception":false,"start_time":"2021-03-14T16:50:53.583035","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def predict_test_with_multiple_models(\n my_models: List[List[torch.nn.Module]],\n my_loaders: List[torch.utils.data.DataLoader],\n predict_func: Callable,\n device: str,\n):\n logits = []\n \n # TTA loop\n for my_loader in my_loaders:\n temp_logits = []\n \n # Loader loop\n for batch in tqdm(my_loader):\n temp_logits_inner = []\n \n # Experiments loop\n for exp_models in my_models:\n logit = np.stack(\n # CV loop\n [predict_func(batch, m, device) for m in exp_models], axis=0\n )#.mean(0)\n temp_logits_inner.append(logit)\n \n temp_logits.append(np.stack(temp_logits_inner, axis=0))\n\n logits.append(np.concatenate(temp_logits, axis=2))\n\n return np.stack(logits, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.502532,"end_time":"2021-03-14T16:50:55.152218","exception":false,"start_time":"2021-03-14T16:50:54.649686","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 512 res"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:50:56.171359Z","iopub.status.busy":"2021-03-14T16:50:56.17066Z","iopub.status.idle":"2021-03-14T16:50:56.173163Z","shell.execute_reply":"2021-03-14T16:50:56.173577Z"},"papermill":{"duration":0.510908,"end_time":"2021-03-14T16:50:56.173707","exception":false,"start_time":"2021-03-14T16:50:55.662799","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"INF_BS = 32","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:50:57.206295Z","iopub.status.busy":"2021-03-14T16:50:57.205661Z","iopub.status.idle":"2021-03-14T16:50:57.240186Z","shell.execute_reply":"2021-03-14T16:50:57.24061Z"},"papermill":{"duration":0.563456,"end_time":"2021-03-14T16:50:57.240763","exception":false,"start_time":"2021-03-14T16:50:56.677307","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"all_test_loaders_512 = []\n\ntest_original = RANZCRDataset(**{\n \"df\":sample_sub,\n \"root\":'test_images_512_512',\n \"path_col\": \"StudyInstanceUID\",\n \"ext\": \".jpeg\",\n \"transforms\":T.ToTensor()\n})\n\nall_test_loaders_512.append(torch.utils.data.DataLoader(\n test_original,\n batch_size=INF_BS,\n drop_last=False,\n shuffle=False,\n num_workers=os.cpu_count() // 2\n))\n\ntest_hf = RANZCRDataset(**{\n \"df\":sample_sub,\n \"root\":'test_images_512_512',\n \"path_col\": \"StudyInstanceUID\",\n \"ext\": \".jpeg\",\n \"transforms\":T.ToTensor(),\n \"augmentations\": albu.HorizontalFlip(p=1.0)\n})\n\nall_test_loaders_512.append(torch.utils.data.DataLoader(\n test_hf,\n batch_size=INF_BS,\n drop_last=False,\n shuffle=False,\n num_workers=os.cpu_count() // 2\n))\n\n# test_tp = RANZCRDataset(**{\n# \"df\":sample_sub,\n# \"root\":'test_images_512_512',\n# \"path_col\": \"StudyInstanceUID\",\n# \"ext\": \".jpeg\",\n# \"transforms\":T.ToTensor(),\n# \"augmentations\": albu.Transpose(p=1.0)\n# })\n\n# all_test_loaders_512.append(torch.utils.data.DataLoader(\n# test_tp,\n# batch_size=INF_BS,\n# drop_last=False,\n# shuffle=False,\n# num_workers=os.cpu_count() // 2\n# ))\n\n# test_tp_vf = RANZCRDataset(**{\n# \"df\":sample_sub,\n# \"root\":'test_images_512_512',\n# \"path_col\": \"StudyInstanceUID\",\n# \"ext\": \".jpeg\",\n# \"transforms\":T.ToTensor(),\n# \"augmentations\": albu.Compose([albu.Transpose(p=1.0), albu.VerticalFlip(p=1.0)])\n# })\n\n# all_test_loaders_512.append(torch.utils.data.DataLoader(\n# test_tp_vf,\n# batch_size=INF_BS,\n# drop_last=False,\n# shuffle=False,\n# num_workers=os.cpu_count() // 2\n# ))\n\n# test_tp_hf = RANZCRDataset(**{\n# \"df\":sample_sub,\n# \"root\":'test_images_512_512',\n# \"path_col\": \"StudyInstanceUID\",\n# \"ext\": \".jpeg\",\n# \"transforms\":T.ToTensor(),\n# \"augmentations\": albu.Compose([albu.Transpose(p=1.0), albu.HorizontalFlip(p=1.0)])\n# })\n\n# all_test_loaders_512.append(torch.utils.data.DataLoader(\n# test_tp_hf,\n# batch_size=INF_BS,\n# drop_last=False,\n# shuffle=False,\n# num_workers=os.cpu_count() // 2\n# ))\n\n\n# test_vf = RANZCRDataset(**{\n# \"df\":sample_sub,\n# \"root\":'test_images_512_512',\n# \"path_col\": \"StudyInstanceUID\",\n# \"ext\": \".jpeg\",\n# \"transforms\":T.ToTensor(),\n# \"augmentations\": albu.VerticalFlip(p=1.0)\n# })\n\n# all_test_loaders_512.append(torch.utils.data.DataLoader(\n# test_vf,\n# batch_size=INF_BS,\n# drop_last=False,\n# shuffle=False,\n# num_workers=os.cpu_count() // 2\n# ))\n\n# test_hf_vf = RANZCRDataset(**{\n# \"df\":sample_sub,\n# \"root\":'test_images_512_512',\n# \"path_col\": \"StudyInstanceUID\",\n# \"ext\": \".jpeg\",\n# \"transforms\":T.ToTensor(),\n# \"augmentations\": albu.Compose([albu.HorizontalFlip(p=1.0), albu.VerticalFlip(p=1.0)])\n# })\n\n# all_test_loaders_512.append(torch.utils.data.DataLoader(\n# test_hf_vf,\n# batch_size=INF_BS,\n# drop_last=False,\n# shuffle=False,\n# num_workers=os.cpu_count() // 2\n# ))\n\ntest_left_rot = RANZCRDataset(**{\n \"df\":sample_sub,\n \"root\":'test_images_512_512',\n \"path_col\": \"StudyInstanceUID\",\n \"ext\": \".jpeg\",\n \"transforms\":T.ToTensor(),\n \"augmentations\": albu.ShiftScaleRotate(\n shift_limit=0,\n scale_limit=0,\n rotate_limit=(15,16),\n p=1.0\n )\n})\n\nall_test_loaders_512.append(torch.utils.data.DataLoader(\n test_left_rot,\n batch_size=INF_BS,\n drop_last=False,\n shuffle=False,\n num_workers=os.cpu_count() // 2\n))\n\ntest_right_rot = RANZCRDataset(**{\n \"df\":sample_sub,\n \"root\":'test_images_512_512',\n \"path_col\": \"StudyInstanceUID\",\n \"ext\": \".jpeg\",\n \"transforms\":T.ToTensor(),\n \"augmentations\": albu.ShiftScaleRotate(\n shift_limit=0,\n scale_limit=0,\n rotate_limit=(-16,-15),\n p=1.0\n )\n})\n\nall_test_loaders_512.append(torch.utils.data.DataLoader(\n test_right_rot,\n batch_size=INF_BS,\n drop_last=False,\n shuffle=False,\n num_workers=os.cpu_count() // 2\n))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T16:50:58.363104Z","iopub.status.busy":"2021-03-14T16:50:58.362511Z","iopub.status.idle":"2021-03-14T17:22:31.804182Z","shell.execute_reply":"2021-03-14T17:22:31.804627Z"},"papermill":{"duration":1894.006269,"end_time":"2021-03-14T17:22:31.804793","exception":false,"start_time":"2021-03-14T16:50:57.798524","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test_logits_512 = predict_test_with_multiple_models(\n models_512,\n all_test_loaders_512,\n cnn_model_predict,\n DEVICE\n)\n","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.620827,"end_time":"2021-03-14T17:22:33.041092","exception":false,"start_time":"2021-03-14T17:22:32.420265","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Blend and create submission.csv"},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T17:22:34.312832Z","iopub.status.busy":"2021-03-14T17:22:34.311309Z","iopub.status.idle":"2021-03-14T17:22:34.31383Z","shell.execute_reply":"2021-03-14T17:22:34.314265Z"},"papermill":{"duration":0.660216,"end_time":"2021-03-14T17:22:34.31441","exception":false,"start_time":"2021-03-14T17:22:33.654194","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def to_rank(input):\n return pd.DataFrame(input).rank().values","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T17:22:35.549292Z","iopub.status.busy":"2021-03-14T17:22:35.548466Z","iopub.status.idle":"2021-03-14T17:22:35.550035Z","shell.execute_reply":"2021-03-14T17:22:35.550488Z"},"papermill":{"duration":0.617501,"end_time":"2021-03-14T17:22:35.550622","exception":false,"start_time":"2021-03-14T17:22:34.933121","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from scipy.special import expit","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T17:22:36.779289Z","iopub.status.busy":"2021-03-14T17:22:36.778516Z","iopub.status.idle":"2021-03-14T17:22:36.782144Z","shell.execute_reply":"2021-03-14T17:22:36.781747Z"},"papermill":{"duration":0.621263,"end_time":"2021-03-14T17:22:36.782257","exception":false,"start_time":"2021-03-14T17:22:36.160994","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test_logits_512.shape","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T17:22:38.026254Z","iopub.status.busy":"2021-03-14T17:22:38.02551Z","iopub.status.idle":"2021-03-14T17:22:38.036879Z","shell.execute_reply":"2021-03-14T17:22:38.036464Z"},"papermill":{"duration":0.643373,"end_time":"2021-03-14T17:22:38.036997","exception":false,"start_time":"2021-03-14T17:22:37.393624","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"test_logits = expit(test_logits_512).mean(0).mean(1) # by TTA and by CV by Exp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_exp_1 = test_logits[0]\nmy_exp_2 = test_logits[1]\nmy_exp_3 = test_logits[2]\nmy_exp_4 = test_logits[3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_exp_1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"blend = (\n# public_pred_1**0.5 + \n# public_pred_3**0.5 +\n# (public_pred_2**0.5) * 2 + \n \n my_exp_1**0.5 + \n my_exp_2**0.5 +\n my_exp_3**0.5 +\n my_exp_3**0.5\n) ","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T17:22:49.9857Z","iopub.status.busy":"2021-03-14T17:22:49.984526Z","iopub.status.idle":"2021-03-14T17:22:50.021825Z","shell.execute_reply":"2021-03-14T17:22:50.02232Z"},"papermill":{"duration":0.656135,"end_time":"2021-03-14T17:22:50.022478","exception":false,"start_time":"2021-03-14T17:22:49.366343","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"sample_sub.iloc[:,1:] = blend\n\n#sample_sub.iloc[:,1:] = result_512\n\nsample_sub","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T17:22:51.286437Z","iopub.status.busy":"2021-03-14T17:22:51.285791Z","iopub.status.idle":"2021-03-14T17:22:51.300157Z","shell.execute_reply":"2021-03-14T17:22:51.299657Z"},"papermill":{"duration":0.66464,"end_time":"2021-03-14T17:22:51.300264","exception":false,"start_time":"2021-03-14T17:22:50.635624","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"sample_sub.nunique(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2021-03-14T17:22:52.559508Z","iopub.status.busy":"2021-03-14T17:22:52.558665Z","iopub.status.idle":"2021-03-14T17:22:53.803631Z","shell.execute_reply":"2021-03-14T17:22:53.803168Z"},"papermill":{"duration":1.888734,"end_time":"2021-03-14T17:22:53.803754","exception":false,"start_time":"2021-03-14T17:22:51.91502","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"!rm -rf test_images_512_512\n\nsample_sub.to_csv('submission.csv', index=False)\n\nos.listdir('./')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}