{"cells":[{"metadata":{"trusted":true,"_uuid":"ecdf7a38b6ff8788044922ec590212b1004fe79e"},"cell_type":"code","source":"%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1924cedcf666887b145ccc8ad8f183e3f408998f"},"cell_type":"code","source":"from typing import List\nimport logging\nfrom typing import Optional\nfrom functools import partial\nfrom typing import Tuple\nfrom typing import Union\n\n\nimport torch.nn as nn\nimport numpy as np\nimport os\nimport pandas as pd\nimport torch\nfrom torch.optim import Adam\nfrom torchvision.models.resnet import BasicBlock\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset\nfrom PIL import Image\nfrom matplotlib import pyplot as plt\nfrom torchvision.models.resnet import ResNet\nfrom sklearn.metrics import roc_auc_score\nfrom torch import Tensor\nfrom torchvision import transforms\nfrom torch.autograd import Variable","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"DATA_FOLDER = '../input'\nLABELS = f'{DATA_FOLDER}/train_labels.csv'\nTRAIN_IMAGES_FOLDER = f'{DATA_FOLDER}/train'\nUSE_GPU = torch.cuda.is_available()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c60226f1207bcf9639317bdc7fc770447e5c225c"},"cell_type":"code","source":"logging.basicConfig(level='INFO')\nlogger = logging.getLogger()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"116c4e4d4da37ec9d36ff419dc27237bb2ff7d3b"},"cell_type":"code","source":"def read_labels(path_to_file: str) -> pd.DataFrame:\n    labels = pd.read_csv(path_to_file)\n    return labels\n\n\ndef format_labels_for_dataset(labels: pd.DataFrame) -> np.array:\n    return labels['label'].values.reshape(-1, 1)\n\n\ndef format_path_to_images_for_dataset(labels: pd.DataFrame, path: str) -> List:\n    return [os.path.join(path, f'{f}.tif') for f in labels['id'].values]\n\n\ndef train_valid_split(df: pd.DataFrame) -> Tuple:\n    limit_df = 50000\n    df = df.sample(n = df.shape[0])\n    df = df.iloc[:limit_df]\n    split = 40000\n    train = df.iloc[:split]\n    valid = df.iloc[:split]\n    return train, valid","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"### STEP 1 - DATASETS"},{"metadata":{"trusted":true,"_uuid":"778a4b6c5d1e0824badd7306c8bbf1364553a02a"},"cell_type":"code","source":"class MainDataset(Dataset):\n    def __init__(self,\n                 x_dataset: Dataset,\n                 y_dataset: Dataset,\n                 x_tfms: Optional = None):\n        self.x_dataset = x_dataset\n        self.y_dataset = y_dataset\n        self.x_tfms = x_tfms\n\n    def __len__(self) -> int:\n        return self.x_dataset.__len__()\n\n    def __getitem__(self, index: int) -> Tuple:\n        x = self.x_dataset[index]\n        y = self.y_dataset[index]\n        if self.x_tfms is not None:\n            x = self.x_tfms(x)\n        return x, y\n    \nclass ImageDataset(Dataset):\n    def __init__(self, paths_to_imgs: List):\n        self.paths_to_imgs = paths_to_imgs\n\n    def __len__(self) -> int:\n        return len(self.paths_to_imgs)\n\n    def __getitem__(self, index: int) -> Image.Image:\n        img = Image.open(self.paths_to_imgs[index])\n        return img\n\n\nclass LabelDataset(Dataset):\n    def __init__(self, labels: List):\n        self.labels = labels\n\n    def __len__(self) -> int:\n        return len(self.labels)\n\n    def __getitem__(self, index: int) -> int:\n        return self.labels[index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32cb4c3451d8657f3038f22e2b10ecd8de50227d"},"cell_type":"code","source":"labels = read_labels(LABELS)\ntrain, valid = train_valid_split(labels)\n\ntrain_labels = format_labels_for_dataset(train)\nvalid_labels = format_labels_for_dataset(valid)\n\ntrain_images = format_path_to_images_for_dataset(train, TRAIN_IMAGES_FOLDER)\nvalid_images = format_path_to_images_for_dataset(valid, TRAIN_IMAGES_FOLDER)\n\ntrain_images_dataset = ImageDataset(train_images)\nvalid_images_dataset = ImageDataset(valid_images)\ntrain_labels_dataset = LabelDataset(train_labels)\nvalid_labels_dataset = LabelDataset(valid_labels)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b81c0c47c4ed580b977f38c80c6e5041f24c9d49"},"cell_type":"markdown","source":"###  STEP 2 - TRANSFORMERS"},{"metadata":{"trusted":true,"_uuid":"9f42b4937675aaef1454d2fbf6fa63220b3f2d7a"},"cell_type":"code","source":"x_tfms = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225]\n    )\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f3a3a593f7dc87450a134556abf48ffd526fadd"},"cell_type":"code","source":"train_dataset = MainDataset(train_images_dataset, train_labels_dataset, x_tfms)\nvalid_dataset = MainDataset(valid_images_dataset, valid_labels_dataset, x_tfms)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9dd26fc5d0a41925027149f960165d87c0e1a18e"},"cell_type":"markdown","source":"### STEP 3 - DATALOADER"},{"metadata":{"trusted":true,"_uuid":"c2fdf7e8809897fd99d6010f4e363031a6d13b31"},"cell_type":"code","source":"shuffle = True\nbatch_size = 512\nnum_workers = 0\ntrain_dataloader = DataLoader(train_dataset, \n                              batch_size=batch_size, \n                              shuffle=shuffle, \n                              num_workers=num_workers)\nvalid_dataloader = DataLoader(valid_dataset, \n                              batch_size=batch_size, \n                              shuffle=False, \n                              num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d29580a7bf02d541c2402dd5bf63445f3c8f63bb"},"cell_type":"markdown","source":"### STEP 4 - MODEL"},{"metadata":{"trusted":true,"_uuid":"5c8aa2ae6a7f5ab31d4e9820cbdc738c824dcdfb"},"cell_type":"code","source":"# we will create custom resnet with 9 layers so it can process 96x96 images\ndef to_gpu(tensor):\n    return tensor.cuda() if USE_GPU else tensor\n\n\ndef create_resnet9_model(output_dim: int = 1) -> nn.Module:\n    model = ResNet(BasicBlock, [1, 1, 1, 1])\n    in_features = model.fc.in_features\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    model.fc = nn.Linear(in_features, output_dim)\n    model = to_gpu(model)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"799947bc958bdad6508c7700a78afc2f382b3c98"},"cell_type":"code","source":"resnet9 = create_resnet9_model(output_dim=1)\nresnet9","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70871546bbb064a9e021c3a0c59dc83fac45e697"},"cell_type":"markdown","source":"### STEP 5 - OPTIMIZER"},{"metadata":{"trusted":true,"_uuid":"338be167c3c298dce62da9e5319b417fddd9474d"},"cell_type":"code","source":"lr = 1e-3\noptimizer = Adam(resnet9.parameters(), lr=lr)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c5be75388cb786ca525c0576d9c4960ee815358"},"cell_type":"markdown","source":"### STEP 6 - LOSS"},{"metadata":{"trusted":true,"_uuid":"deb9a6e2a8672e96cd7352bffcb438694d98b95d"},"cell_type":"code","source":"loss = nn.BCEWithLogitsLoss()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c5f32a2e61de6f396dc67b953ae635ce10a5349"},"cell_type":"markdown","source":"### STEP 7 - TRAINER / PREDICTOR / TRIGGERS"},{"metadata":{"trusted":true,"_uuid":"386c80632b7cbcc4be6c937221de1119326ead98"},"cell_type":"code","source":"def train_one_epoch(model,\n                    train_dataloader,\n                    valid_dataloader,\n                    loss,\n                    optimizer,\n                    loss_writer_train,\n                    loss_writer_valid,\n                    do_step_trigger,\n                    valid_loss_trigger,\n                    train_loss_trigger,\n                    ):\n    model.train()\n    y_true_train, y_hat_train = [], []\n    for iteration, (x, y) in enumerate(train_dataloader):\n        x = Variable(T(x), requires_grad=True)\n        y = Variable(T(y), requires_grad=True)\n        output = model(x)\n        y_true_train.append(to_numpy(y))\n        y_hat_train.append(to_numpy(output))\n        loss_values = loss(output, y)\n        loss_values.backward()\n        if do_step_trigger(iteration):\n            optimizer.step()\n            optimizer.zero_grad()\n        if train_loss_trigger(iteration):\n            auc_writer(y_true_train, y_hat_train, iteration)\n            y_true_train, y_hat_train = [], []\n        if valid_loss_trigger(iteration):\n            y_true, y_hat = predict(model, valid_dataloader)\n            auc_writer(y_true, y_hat, iteration)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"470d696d275572be72cb062392444b23445e745d"},"cell_type":"code","source":"def predict(model, dataloader):\n    model.eval()\n    y_true, y_hat = [], []\n    with torch.no_grad():\n        for x, y in dataloader:\n            x = Variable(T(x))\n            y = Variable(T(y))\n            output = model(x)\n            y_true.append(to_numpy(y))\n            y_hat.append(to_numpy(output))\n    return y_true, y_hat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24091614085c019a5e6d670d8142507568807d26"},"cell_type":"code","source":"def iteration_trigger(iteration, every_x_iterations):\n    if every_x_iterations == 1:\n        return True\n    elif iteration > 0 and iteration % every_x_iterations == 0:\n        return True\n    else:\n        return False\n\n\ndef init_triggers(step=1, valid=10, train=10):\n    do_step_trigger = partial(iteration_trigger, every_x_iterations=step)\n    valid_loss_trigger = partial(iteration_trigger, every_x_iterations=valid)\n    train_loss_trigger = partial(iteration_trigger, every_x_iterations=train)\n    return do_step_trigger, valid_loss_trigger, train_loss_trigger","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17cc5a3226890fa035bcfbd9c045b6cb1f6b477a"},"cell_type":"code","source":"def auc_writer(y_true, y_hat, iteration):\n    try:\n        auc = roc_auc_score(np.vstack(y_true), np.vstack(y_hat))\n    except:\n        auc = -1\n    logger.info(f'iteration: {iteration}, auc: {auc}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10d9128afe353fbdbb7c1b6abcc2188b900afed4"},"cell_type":"code","source":"def T(tensor):\n    if not torch.is_tensor(tensor):\n        tensor = torch.FloatTensor(tensor)\n    else:\n        tensor = tensor.type(torch.FloatTensor)\n    if USE_GPU:\n        tensor = to_gpu(tensor)\n    return tensor\n\n\ndef to_numpy(tensor: Union[Tensor, Image.Image, np.array]) -> np.ndarray:\n    if type(tensor) == np.array or type(tensor) == np.ndarray:\n        return np.array(tensor)\n    elif type(tensor) == Image.Image:\n        return np.array(tensor)\n    elif type(tensor) == Tensor:\n        return tensor.cpu().detach().numpy()\n    else:\n        raise ValueError(msg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99873a14fa17bef9023b2e81f84e92146ab4a049"},"cell_type":"code","source":"loss_writer_train = auc_writer\nloss_writer_valid = auc_writer\n\n# backpropagate - every iter\n# calculate AUC on valid - every 20 iters \n# AUC on train - every 10 iters\ndo_step_trigger, valid_loss_trigger, train_loss_trigger = init_triggers(1, 20, 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08aca6b82c27a9c3f55caa6849410e66e867a467"},"cell_type":"code","source":"resnet9 = train_one_epoch(\n    resnet9,\n    train_dataloader,\n    valid_dataloader,\n    loss,\n    optimizer,\n    loss_writer_train,\n    loss_writer_valid,\n    do_step_trigger,\n    valid_loss_trigger,\n    train_loss_trigger,\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"012aee8047dc2a4786a26e53ff9a9decd8244de9"},"cell_type":"markdown","source":"### STEP 8 - PREDICTIONS EXTRACTION"},{"metadata":{"trusted":true,"_uuid":"074797140b83c341e8527ae2818bde43947329b6"},"cell_type":"code","source":"y_true, y_hat = predict(resnet9, valid_dataloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4ba698f0bc790360a34c1218c3b11bc4393bb17"},"cell_type":"code","source":"predictions = pd.DataFrame(\n    list(\n        zip(\n            valid_labels.reshape(-1), \n            np.vstack(y_hat).reshape(-1),\n            valid_images\n        )\n    ), \n     columns=['true', 'pred', 'files'])\npredictions.head(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"133bd1628c674ed135938d5ae64be29ea44ecb19"},"cell_type":"markdown","source":"### STEP 9 - ERRORS ANALYSIS"},{"metadata":{"trusted":true,"_uuid":"21f8093ec5fbd913d51240189f8cdafd1b0877a7"},"cell_type":"code","source":"max_charts = 60\ndef implot(files, w=2, h=2, cols=12):\n    rows = len(files) / cols + 1\n    images = [Image.open(f) for f in files]\n    plt.figure(figsize = (cols * w, rows * h))\n    plt.tight_layout()\n    for chart, img in enumerate(images, 1):\n        ax = plt.subplot(rows, cols, chart)\n        ax.imshow(np.array(img))\n        ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"46fcae92b1fe5e8eb13f4272757425d9c4120681"},"cell_type":"markdown","source":"#### FALSE NEGATIVE"},{"metadata":{"trusted":true,"_uuid":"9b8be53de445b7c0aa483ed10adbf8a14f058c46"},"cell_type":"code","source":"files = predictions[predictions['true']==1].sort_values('pred')['files'].values[:max_charts]\nimplot(files)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8c20c5d3175639650353eee3309f368498cd97c2"},"cell_type":"markdown","source":"#### FALSE POSITIVE"},{"metadata":{"trusted":true,"_uuid":"bc634dc8f88dd962bfbbf4e631663ab141dcb777"},"cell_type":"code","source":"files = predictions[predictions['true']==0].sort_values('pred', ascending=False)['files'].values[:max_charts]\nimplot(files)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8b72362d0f8c2d3ab1a0a84063ad3a18e6c27d89"},"cell_type":"markdown","source":"#### TRUE POSITIVE"},{"metadata":{"trusted":true,"_uuid":"c3f3440fb329fd96938bb78b5d3de813b79d436d"},"cell_type":"code","source":"files = predictions[predictions['true']==1].sort_values('pred', ascending=False)['files'].values[:max_charts]\nimplot(files)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6368d7eee2f8009b817ac5feb5ecd57ceb52f9d5"},"cell_type":"markdown","source":"#### TRUE NEGATIVE"},{"metadata":{"trusted":true,"_uuid":"a175ad5683cb044c354389cc064616884219e4d3"},"cell_type":"code","source":"files = predictions[predictions['true']==0].sort_values('pred', ascending=True)['files'].values[:max_charts]\nimplot(files)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}