{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Histopathologic Cancer Detection"},{"metadata":{"trusted":true},"cell_type":"code","source":"from typing import List\nimport logging\nfrom typing import Optional\nfrom functools import partial\nfrom typing import Tuple\nfrom typing import Union\n\n\nimport torch.nn as nn\nimport numpy as np\nimport os\nimport pandas as pd\nimport torch\nfrom torch.optim import Adam\nfrom torchvision.models.resnet import BasicBlock\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset\nfrom PIL import Image\nfrom matplotlib import pyplot as plt\nfrom torchvision.models.resnet import ResNet\nfrom sklearn.metrics import roc_auc_score\nfrom torch import Tensor\nfrom torchvision import transforms\nfrom torch.autograd import Variable\nimport albumentations as A","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DATA_FOLDER = '../input/histopathologic-cancer-detection'\nLABELS = f'{DATA_FOLDER}/train_labels.csv'\nTRAIN_IMAGES_FOLDER = f'{DATA_FOLDER}/train'\nSAMPLE_SUBMISSION = f'{DATA_FOLDER}/sample_submission.csv'\nUSE_GPU = torch.cuda.is_available() \n\n\nDATA_FOLDER\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"USE_GPU","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logging.basicConfig(level='INFO') #логгирование = запись всех действий для выявления ошибок\nlogger = logging.getLogger()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = pd.read_csv(LABELS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels  #картинки с \"cancer - no cancer\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Метод для конвертации дата фрейма ответов в нумпай массив => делаем массив лейблов и индексов\ndef format_labels_for_data_set(labels):\n    return (labels['label'].values.reshape(-1,1))\n\n# Делим train на тренировочную и валидационную выборки \ndef train_valid_split(df, split_percent, limit_df= 10000 ):\n#     limit_d -  count of images\n    df = df.sample(n = df.shape[0])\n    df = df.iloc[:limit_df]\n    split = round(limit_df * split_percent / 100)\n    train = df.iloc[:split] #параметры деления на группы\n    valid = df.iloc[split:]\n    return (train, valid)\n\n# возвращает полный путь к картинкам из labels sample\ndef format_path_to_images_for_dataset(labels, path):\n    return [os.path.join(path, f'{f}.tif') for f in labels['id'].values]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"STEP 1 - DATASETS"},{"metadata":{"trusted":true},"cell_type":"code","source":"# класс, в котором определяется исходный DataSet и делается масштабирование исходных данных\nclass MainDataset(Dataset):\n    def __init__(self, x_dataset, y_dataset, x_tfms):\n        self.x_dataset = x_dataset\n        self.y_dataset = y_dataset\n        self.x_tfms = x_tfms \n        \n    def __len__(self):\n        return self.x_dataset.__len__() \n        \n    def __getitem__(self, index):   #Возвращает уже x_tfms от картинки (её тензорный вид (массиввы массивов))+лейблы\n        x = self.x_dataset[index]   #когда делаем a[10], где a = MainDataset()\n        y = self.y_dataset[index]\n        if x_tfms is not None:\n            x = self.x_tfms(x)\n        return x, y\n\n# возвращает картинку (с учетом ее полного пути) по индексу\nclass ImageDataset(Dataset):\n    def __init__(self, path_to_image):\n        self.path_to_image = path_to_image\n    \n    def __len__(self):\n        return len(self.path_to_image)\n    \n    def __getitem__(self, index):\n        img = Image.open(self.path_to_image[index]) \n        \n        # Compose a complex augmentation pipeline => задаем аугментацию = расширяем обучающую выборку\n        augmentation_pipeline = A.Compose([\n            A.HorizontalFlip(p = 0.5), # apply horizontal flip to 50% of images\n            A.OneOf(\n                [\n                    # apply one of transforms to 50% of images\n                    A.RandomContrast(), # apply random contrast\n                    A.RandomGamma(), # apply random gamma\n                    A.RandomBrightness(limit = -0.1), # apply random brightness\n                ],\n                p = 1\n            ),\n            A.ShiftScaleRotate(p = 0.5) #афинные преобразования \n        ],\n        p = 1)\n        \n        image_aug = augmentation_pipeline(image = np.array(img))['image'] # apply pipeline to image\n        image = Image.fromarray(image_aug, 'RGB') #why RGB??\n        return image\n\n\n# возвращает label по индексу\nclass LabelDataset(Dataset):\n    def __init__(self, labels):\n        self.labels = labels\n    \n    def __len__(self):\n        return len(labels)\n    \n    def __getitem__(self, index):\n        return self.labels[index]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = pd.read_csv(LABELS)\nsample_submission = pd.read_csv(SAMPLE_SUBMISSION)\n\ntrain, valid = train_valid_split(labels, 70) #заполнение train массивом для тренировки, valid - for validation\n\ntrain_labels = format_labels_for_data_set(train)\nvalid_labels = format_labels_for_data_set(valid)\n\ntrain_images = format_path_to_images_for_dataset(train, TRAIN_IMAGES_FOLDER)\nvalid_images = format_path_to_images_for_dataset(valid, TRAIN_IMAGES_FOLDER)\n\ntrain_images_dataset = ImageDataset(train_images) #массив путей для images\nvalid_images_dataset = ImageDataset(valid_images)\ntrain_labels_dataset = LabelDataset(train_labels)\nvalid_labels_dataset = LabelDataset(valid_labels)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_images_dataset[101]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Посмотрим на картинки с аугментацией\ndef implot(dataset, w=2, h=2, cols=12, max_charts = 24 ):\n    rows = (max_charts) / cols + 1\n    images = [dataset[3] for i in range(max_charts)]\n    plt.figure(figsize = (cols * w, rows * h))\n    plt.tight_layout()\n    for chart, img in enumerate(images, 1):\n        ax = plt.subplot(rows, cols, chart)\n        ax.imshow(np.array(img))\n        ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# implot(train_images_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"STEP 2 - TRANSFORMERS"},{"metadata":{"trusted":true},"cell_type":"code","source":"# зададим форматирование и перевод в тензорный вид наших тренировочных данных \n        # тензор нужен для представления входных параметров\n\nx_tfms = transforms.Compose([transforms.ToTensor(), \n                             transforms.Normalize(\n                                 mean=[0.485, 0.456, 0.406],\n                                 std=[0.229, 0.224, 0.225]\n                             )\n                            ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x_tfms","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# определим объеккты из данных и ответов для загрузки в data loader\ntrain_dataset = MainDataset(train_images_dataset, train_labels_dataset, x_tfms)\nvalid_dataset = MainDataset(valid_images_dataset, valid_labels_dataset, x_tfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x_tfms(train_images_dataset[0])\n# train_dataset[0][0][0][0][0] #--> 5мерный массив?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"STEP 3 - DATALOADER"},{"metadata":{"trusted":true},"cell_type":"code","source":"# загружаем данные в data loader, определяем число батчей\nshuffle = True\nbatch_size = 512\nnum_workers = 0\n\ntrain_dataloader = DataLoader(train_dataset, batch_size = batch_size, shuffle = shuffle, num_workers = num_workers)\nvalid_dataloader = DataLoader(valid_dataset, batch_size = batch_size, shuffle = shuffle, num_workers = num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#DataLoader","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"STEP 4 - MODEL"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef to_gpu(tensor):\n    return tensor.cuda() if USE_GPU else tensor\n\ndef create_resnet9_model(output_dim: int = 1) -> nn.Module:\n    model = ResNet(BasicBlock, [1, 1, 1, 1]) #ResNet - наша модель?\n    # размер входящей картинки\n    in_features = model.fc.in_features\n    # output size = 1X1\n    model.avgpool = nn.AdaptiveAvgPool2d(1)\n    model.fc = nn.Linear(in_features, output_dim)\n    model = to_gpu(model)\n    return model\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet9 = create_resnet9_model()\n# resnet9 #=> 4 слоя","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"STEP 5 - OPTIMIZER"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nlr = 1e-3\noptimizer = Adam(resnet9.parameters(), lr) #оптимизация весов?","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"STEP 6 - LOSS"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nloss = nn.BCEWithLogitsLoss() #функция ошибок\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"STEP 7 - TRAINER / PREDICTOR / TRIGGERS"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef auc_writer(y_true, y_predicted, iteration):\n    try:\n        score = roc_auc_score(np.vstack(y_true), np.vstack(y_predicted)) # \n    except:\n        score = -1\n    print(f'iteration: {iteration}, roc_auc: {score}')\n    logger.info(f'iteration: {iteration}, roc_auc: {score}')    \n    \nloss_writer_train = auc_writer\nloss_writer_valid = auc_writer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef predict(model, dataloader):\n    model.eval()\n    y_true, y_hat = [], []\n    \n    for x, y in dataloader:\n        x = Variable(T(x))\n        y = Variable(T(y))\n        output = model(x)\n        \n        y_true.append(to_numpy(y))\n        y_hat.append(to_numpy(output))\n    \n    return y_true, y_hat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ПРиведение тензора в вид для Variable() (мб транспонирование)\ndef T(tensor):\n    if not torch.is_tensor(tensor):\n        tensor = torch.FloatTensor(tensor)\n    else:\n        tensor = tensor.type(torch.FloatTensor)\n    if USE_GPU:\n        tensor = to_gpu(tensor)\n    return tensor\n\n# Всё в нампи или ошибку\ndef to_numpy(tensor):\n    if type(tensor) == np.array or type(tensor) == np.ndarray:\n        return np.array(tensor)\n    elif type(tensor) == Image.Image:\n        return np.array(tensor)\n    elif type(tensor) == Tensor:\n        return tensor.cpu().detach().numpy()\n    else:\n        raise ValueError(msg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# прерыватель каждые n попыток \ndef iteration_trigger(iteration, every_x_iteration):\n    if every_x_iteration == 1:\n        return True\n    elif iteration > 0 and iteration % every_x_iteration == 0:\n        return True\n    else:\n        return False\n    \n\n\ndef init_triggers(step = 1, train = 10, valid = 10):\n    do_step_trigger = partial(iteration_trigger, every_x_iteration = step)\n    train_loss_trigger = partial(iteration_trigger, every_x_iteration = train)\n    valid_loss_trigger = partial(iteration_trigger, every_x_iteration = valid)\n    \n    return do_step_trigger, train_loss_trigger, valid_loss_trigger\n\n\ndo_step_trigger, train_loss_trigger, valid_loss_trigger = init_triggers(1, 12, 24)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef train_one_epoch(model, \n                    train_data_loader, \n                    valid_data_loader, \n                    loss, \n                    optimizer, \n                    loss_writer_train, \n                    loss_writer_valid,\n                    do_step_trigger,\n                    train_loss_trigger,\n                    valid_loss_trigger):\n    \n    y_true_train, y_hat_train = [], []\n    for iteration, (x, y) in enumerate(train_data_loader):\n        x_train = Variable(T(x), requires_grad = True)\n        y_train = Variable(T(y), requires_grad = True)\n        \n        output = model(x_train)\n        y_true_train.append(to_numpy(y_train))\n        y_hat_train.append(to_numpy(output))\n        loss_values = loss(output, y_train)\n        loss_values.backward()\n        \n        #делаем шаг на каждой итерации и сбрасываем градиент\n        if do_step_trigger(iteration):\n            optimizer.step()\n            optimizer.zero_grad()\n        \n        # проверяем, если итерация кратна train_step = 13, то тогда записываем в лог значение roc_auc\n        if train_loss_trigger(iteration):\n            print('train_loss_trigger: ')\n            loss_writer_train(y_true_train, y_hat_train, iteration)\n            y_true_train, y_hat_train = [], []\n        \n        # проверяем, если итерация кратна valid_step = 20, то тогда записываем в лог значение roc_auc\n        if valid_loss_trigger(iteration):\n            print('valid_loss_trigger:')\n            y_true_valid, y_hat_valid = predict(model, valid_data_loader)\n            loss_writer_valid(y_true_valid, y_hat_valid, iteration)\n        \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nresnet9 = train_one_epoch(resnet9, \n                    train_dataloader, \n                    valid_dataloader, \n                    loss, \n                    optimizer, \n                    loss_writer_train, \n                    loss_writer_valid,\n                    do_step_trigger,\n                    train_loss_trigger,\n                    valid_loss_trigger)\nresnet9 = train_one_epoch(resnet9, \n                    train_dataloader, \n                    valid_dataloader, \n                    loss, \n                    optimizer, \n                    loss_writer_train, \n                    loss_writer_valid,\n                    do_step_trigger,\n                    train_loss_trigger,\n                    valid_loss_trigger)\nresnet9 = train_one_epoch(resnet9, \n                    train_dataloader, \n                    valid_dataloader, \n                    loss, \n                    optimizer, \n                    loss_writer_train, \n                    loss_writer_valid,\n                    do_step_trigger,\n                    train_loss_trigger,\n                    valid_loss_trigger)\nresnet9 = train_one_epoch(resnet9, \n                    train_dataloader, \n                    valid_dataloader, \n                    loss, \n                    optimizer, \n                    loss_writer_train, \n                    loss_writer_valid,\n                    do_step_trigger,\n                    train_loss_trigger,\n                    valid_loss_trigger)\nresnet9 = train_one_epoch(resnet9, \n                    train_dataloader, \n                    valid_dataloader, \n                    loss, \n                    optimizer, \n                    loss_writer_train, \n                    loss_writer_valid,\n                    do_step_trigger,\n                    train_loss_trigger,\n                    valid_loss_trigger)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"STEP 8 - TEST PREDICTIONS "},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_IMAGES_FOLDER = f'{DATA_FOLDER}/test/'\n\n# преобразуем исходные данные сначала в Image, затем в Tensor\n\n#сделаем функцию, которая возвращает список названий картинок в папке test\ndef test_image_collection(directory: str) -> List:\n    images_name = []\n    for filename in os.listdir(directory):\n        images_name.append(TEST_IMAGES_FOLDER + filename)\n    return(images_name)\n\ntest_image = test_image_collection(TEST_IMAGES_FOLDER)\ntest_images_dataset = ImageDataset(test_image)    \n\n# зададим форматирование и перевод в тензорный вид наших тестовых данных \nclass TestDataset(Dataset):\n    def __init__(self, x_dataset: Dataset, x_tfms: Optional = None):\n        self.x_dataset = x_dataset\n        self.x_tfms = x_tfms\n        \n    def __len__(self) -> int:\n        return self.x_dataset.__len__() \n        \n    def __getitem__(self, index: int) -> Tuple:\n        x = self.x_dataset[index]\n        if x_tfms is not None:\n            x = self.x_tfms(x)\n        return x\n    \ntest_dataset = TestDataset(test_images_dataset, x_tfms)    \n\n# загружаем данные в data loader, определяем число батчей\nbatch_size = 512\nnum_workers = 0\nshuffle = False\n\ntest_dataloader = DataLoader(test_dataset, batch_size = batch_size, shuffle = shuffle, num_workers = num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# определим функцию предсказания для тестовой выборки\ndef predict_test(model, dataloader):\n    model.eval()\n    y_hat = []\n    \n    for x in dataloader:\n        x = Variable(T(x))\n        output = model(x)\n        \n        y_hat.append(to_numpy(output))\n    return y_hat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# сделаем предсказания для тестовой выборки\ny_hat_test = predict_test(resnet9, test_dataloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# запишем ответы в DataFrame\npredictions = pd.DataFrame(\n    list(\n        zip(\n            test_image,\n            np.vstack(y_hat_test).reshape(-1)\n        )\n    ), \n     columns=['id', 'label'])\npredictions['id'] = predictions['id'].apply(lambda x: x.split('/')[-1].split('.')[0]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"STEP 9 - ERRORS ANALYSIS"},{"metadata":{"trusted":true},"cell_type":"code","source":"# #определим метод для отображения картинок\n# max_charts = 60\n# def implot_errors(files, w=2, h=2, cols=12):\n#     rows = len(files) / cols + 1\n#     images = [Image.open(f) for f in files]\n#     plt.figure(figsize = (cols * w, rows * h))\n#     plt.tight_layout()\n#     for chart, img in enumerate(images, 1):\n#         ax = plt.subplot(rows, cols, chart)\n#         ax.imshow(np.array(img))\n#         ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # сделаем таблицу с предсказаниями и реальными значениями на валидационной выборке\n# y_true, y_hat = predict(resnet9, valid_dataloader)\n\n# predictions_comparison = pd.DataFrame(\n#     list(\n#         zip(\n#             valid_labels.reshape(-1), \n#             np.vstack(y_hat).reshape(-1),\n#             valid_images\n#         )\n#     ), \n#      columns=['true', 'pred', 'files'])\n\n# predictions_comparison.head(3)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"FALSE NEGATIVE"},{"metadata":{"trusted":true},"cell_type":"code","source":"# files = predictions_comparison[predictions_comparison['true']==1].sort_values('pred')['files'].values[:max_charts]\n# implot_errors(files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"FALSE POSITIVE"},{"metadata":{"trusted":true},"cell_type":"code","source":"# files = predictions_comparison[predictions_comparison['true']==0].sort_values('pred', ascending=False)['files'].values[:max_charts]\n# implot_errors(files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"TRUE POSITIVE"},{"metadata":{"trusted":true},"cell_type":"code","source":"# files = predictions_comparison[predictions_comparison['true']==1].sort_values('pred', ascending=False)['files'].values[:max_charts]\n# implot_errors(files)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"TRUE NEGATIVE"},{"metadata":{"trusted":true},"cell_type":"code","source":"# files = predictions_comparison[predictions_comparison['true']==0].sort_values('pred', ascending=True)['files'].values[:max_charts]\n# implot_errors(files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}