{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport glob\nimport copy\nimport cv2\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import roc_auc_score, precision_score, recall_score, f1_score, accuracy_score\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torchvision\nfrom torchvision import models, transforms, utils\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom pretrainedmodels.models import se_resnext50_32x4d, se_resnext101_32x4d\nfrom torch.utils.tensorboard import SummaryWriter\nfrom torchsummary import summary","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"SEED = 233\nCROP_SIZE = 96\nRESIZE_SIZE = 224\nBATCH_SIZE = 64\nTEST_BATCH_SIZE = 64\nMAX_EPOCH = 15\n\nTRAIN_DIR = \"./train/\"\nTEST_DTR = \"./test/\"\n\nTRAIN_CSV = pd.read_csv(\"train_labels.csv\")\nTEST_CSV = pd.read_csv(\"sample_submission.csv\")\nWSI_CSV = pd.read_csv(\"patch_id_wsi.csv\")\n\nDEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef seed_everything(seed=SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n\n\nseed_everything()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef ImageLoader(path):\n    \"\"\"\n    load image as PIL.Image (RGB)\n    \"\"\"\n    return Image.open(path).convert('RGB')\n\n\ndef read_image_list(csv_data, data_dir):\n    \"\"\"\n    get image path list, return a list of path\n    \"\"\"\n    image_list = [os.path.join(data_dir, '{}.tif'.format(e_i)) for e_i in csv_data['id'].values]\n    return image_list\n\n\ndef read_label_list(csv_data):\n    \"\"\"\n    get image label list, return a list of label(0,1)\n    \"\"\"\n    return csv_data['label'].values.reshape(-1, 1)\n\n\ndef get_mean_std():\n    \"\"\"\n    calculate (avr std) of training image\n    \"\"\"\n    if not os.path.exists(\"train_mean_std.npy\"):\n        print(\"Start computing statistics of 220025 images\")\n        dark_th = 10 / 255\n        bright_th = 245 / 255\n        too_dark_idx = []\n        too_bright_idx = []\n\n        x_tot = np.zeros(3)\n        x2_tot = np.zeros(3)\n        counted_ones = 0\n\n        for f_path in read_image_list(TRAIN_CSV, TRAIN_DIR):\n            # norm image\n            imagearray = np.array(ImageLoader(f_path)).reshape(-1, 3) / 255\n            if imagearray.max() < dark_th:  # image too dark?\n                too_dark_idx.append(f_path)\n                continue\n            if imagearray.min() > bright_th:  # image too light?\n                too_bright_idx.append(f_path)\n                continue\n\n            x_tot += imagearray.mean(axis=0)\n            x2_tot += (imagearray ** 2).mean(axis=0)\n            counted_ones += 1\n\n        channel_avr = x_tot / counted_ones\n        channel_std = np.sqrt(x2_tot / counted_ones - channel_avr ** 2)\n        np.save(\"train_mean_std.npy\", np.append(channel_avr, channel_std))\n\n        print(\"Computing finished: {} images\\n\".format(counted_ones), \"-\" * 20)\n        return channel_avr, channel_std\n\n    else:\n        print(\"-\" * 30, \"\\nReading existed file\")\n        avr_std = np.load(\"train_mean_std.npy\")\n        channel_avr = avr_std[:3]\n        channel_std = avr_std[3:]\n        return channel_avr, channel_std\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef get_x_trans_change():\n\n    # mean=[0.485, 0.456, 0.406]\n    # std=[0.229, 0.224, 0.225]\n\n    all_mean, all_std = get_mean_std()\n\n    x_trans = transforms.Compose([\n        transforms.CenterCrop(CROP_SIZE),\n        transforms.Resize((RESIZE_SIZE, RESIZE_SIZE)),\n        transforms.RandomChoice([\n            transforms.ColorJitter(brightness=0.5),\n            transforms.ColorJitter(contrast=0.5),\n            transforms.ColorJitter(saturation=0.5),\n            transforms.ColorJitter(hue=0.5),\n            transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),\n            transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.3),\n            transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.5),\n        ]),\n        transforms.RandomChoice([\n            transforms.RandomRotation((0, 0)),\n            transforms.RandomHorizontalFlip(p=1),\n            transforms.RandomVerticalFlip(p=1),\n            transforms.RandomRotation((90, 90)),\n            transforms.RandomRotation((180, 180)),\n            transforms.RandomRotation((270, 270)),\n            transforms.Compose([\n                transforms.RandomHorizontalFlip(p=1),\n                transforms.RandomRotation((90, 90)),\n            ]),\n            transforms.Compose([\n                transforms.RandomHorizontalFlip(p=1),\n                transforms.RandomRotation((270, 270)),\n            ])\n        ]),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=all_mean, std=all_std)\n    ])\n    return x_trans\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef tta_test():\n\n    all_mean, all_std = get_mean_std()\n\n    change_01 = transforms.Compose([\n        transforms.CenterCrop(CROP_SIZE),\n        transforms.Resize((RESIZE_SIZE, RESIZE_SIZE))\n    ])\n\n    change_02 = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize(mean=all_mean, std=all_std)\n    ])\n\n    change_list = [\n        transforms.ColorJitter(brightness=0.5),\n        transforms.ColorJitter(contrast=0.5),\n        transforms.ColorJitter(saturation=0.5),\n        transforms.ColorJitter(hue=0.5),\n\n        transforms.RandomHorizontalFlip(p=1),\n        transforms.RandomVerticalFlip(p=1),\n\n        transforms.RandomRotation((0, 0)),\n        transforms.RandomRotation((90, 90)),\n        transforms.RandomRotation((180, 180)),\n        transforms.RandomRotation((270, 270)),\n    ]\n\n    x_all = transforms.Lambda(\n        lambda image:\n        torch.stack([change_02(each(change_01(image))) for each in change_list])\n    )\n\n    return x_all\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef wsi_kfold_split():\n    not_in_wsi = TRAIN_CSV.set_index('id').drop(WSI_CSV.id)\n\n    grouped_l = list(WSI_CSV.groupby(by='wsi'))\n\n    grouped_normal = grouped_l[:128]  # normal wsi\n    grouped_tumor = grouped_l[128:]  # tumor wsi\n\n    random.shuffle(grouped_normal)\n    random.shuffle(grouped_tumor)\n\n    k_f_data = {}\n\n    kf_5 = KFold(n_splits=5, shuffle=True, random_state=SEED)\n    not_in_wsi_kf = list(kf_5.split(not_in_wsi))\n\n    for k in range(5):\n        v_normal = grouped_normal[int(k / 5 * len(grouped_normal)):\n                                  int((k + 1) / 5 * len(grouped_normal))]\n        v_tumor = grouped_tumor[int(k / 5 * len(grouped_tumor)):\n                                int((k + 1) / 5 * len(grouped_tumor))]  # validation\n\n        t_normal = [_ for _ in grouped_normal if _ not in v_normal]\n        t_tumor = [_ for _ in grouped_tumor if _ not in v_tumor]  # train\n\n        temp_v = [_v[1] for _v in v_normal] + [_v[1] for _v in v_tumor]\n        temp_t = [_t[1] for _t in t_normal] + [_t[1] for _t in t_tumor]\n\n        random.shuffle(temp_t)\n        random.shuffle(temp_v)\n\n        wsi_k_t = pd.concat(temp_t)  # wsi train\n        wsi_k_v = pd.concat(temp_v)  # wsi valid\n\n        img_k_train = pd.merge(wsi_k_t, TRAIN_CSV, how=\"inner\",\n                               left_on=\"id\", right_on=\"id\").drop([\"wsi\"], axis=1)\n\n        img_k_valid = pd.merge(wsi_k_v, TRAIN_CSV, how=\"inner\",\n                               left_on=\"id\", right_on=\"id\").drop([\"wsi\"], axis=1)\n\n        # not in WSI\n        img_k_train = pd.concat([not_in_wsi.iloc[not_in_wsi_kf[k][0]].reset_index(),\n                                 img_k_train])\n        img_k_valid = pd.concat([not_in_wsi.iloc[not_in_wsi_kf[k][1]].reset_index(),\n                                 img_k_valid])\n\n        k_f_data[k] = {\"train\": img_k_train,\n                       \"val\": img_k_valid}\n\n    return k_f_data\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create dataset ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass MyDataset(Dataset):\n    def __init__(self, csv_file, data_dir, transform=None, loader=ImageLoader):\n        self.image_data = read_image_list(csv_file, data_dir)\n        self.label_data = read_label_list(csv_file)\n\n        self.transform = transform\n        self.loader = loader\n\n    def __getitem__(self, index):\n        x = self.image_data[index]\n        label = self.label_data[index]\n\n        img = self.loader(x)\n        if self.transform is not None:\n            img = self.transform(img)\n        return img, label\n\n    def __len__(self):\n        return len(self.image_data)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRANS = get_x_trans_change()\nTRANS_test = tta_test()\n\nKF_Dataset = {}\nKF_length = {}\nk_wsi_csv = wsi_kfold_split()\n\nfor cnt in range(5):\n    print(cnt + 1)\n\n    train_part = k_wsi_csv[cnt][\"train\"]\n    val_part = k_wsi_csv[cnt][\"val\"]\n\n    train_data = MyDataset(csv_file=train_part, data_dir=TRAIN_DIR,\n                           transform=TRANS, loader=ImageLoader)\n    train_data_loader = DataLoader(train_data, batch_size=BATCH_SIZE,\n                                   shuffle=True, num_workers=4)\n\n    valid_data = MyDataset(csv_file=val_part, data_dir=TRAIN_DIR,\n                           transform=TRANS, loader=ImageLoader)\n    valid_data_loader = DataLoader(valid_data, batch_size=BATCH_SIZE,\n                                   shuffle=True, num_workers=4)\n\n    KF_Dataset[cnt] = {\"train\": train_data_loader,\n                       \"val\": valid_data_loader}\n    KF_length[cnt] = {\"train\": len(train_data),\n                      \"val\": len(valid_data)}\n\n    print(\"-\" * 30, \"\\nTrain data size: {}\\nValid data size: {}\\n\".format(\n        len(train_part), len(val_part)))\n    print(\"Train data batch: {}\\nValid data batch: {}\\n\".format(\n        len(train_data_loader), len(valid_data_loader)))\n\ntest_data = MyDataset(\n    csv_file=TEST_CSV,\n    data_dir=TEST_DTR,\n    transform=TRANS_test,\n    loader=ImageLoader\n)\n\ntest_data_loader = DataLoader(\n    test_data,\n    batch_size=TEST_BATCH_SIZE,\n    num_workers=4\n)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build Net","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass SE_ResNext_50(nn.Module):\n\n    def __init__(self, ):\n        super(SE_ResNext_50, self).__init__()\n\n        model = se_resnext50_32x4d()\n        self.model_layer = nn.Sequential(*list(model.children())[:-1])\n        self.linear_layer = nn.Linear(2048, 1)\n\n    def forward(self, x):\n        x = self.model_layer(x)  # [-1, 2048, 1, 1]\n\n        batch = x.shape[0]\n        conc = x.view(batch, -1)\n        out = self.linear_layer(conc)\n\n        return out\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Net","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef train(max_epoch=MAX_EPOCH):\n    for idx in range(5):\n        print(\"CV {}/5 starts!!!\".format(idx + 1))\n        data_loader = KF_Dataset[idx]\n\n        net = SE_DenseNet169_plus()\n        net.to(DEVICE)\n\n        optimizer = optim.Adam(net.parameters(), lr=1e-4)\n        criterion = nn.BCEWithLogitsLoss()\n        scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=2)\n\n        best_model_wts = copy.deepcopy(net.state_dict())\n        best_acc = 0.0\n        best_epoch = 0\n\n        for epoch in range(max_epoch):\n            print('Epoch {}/{}\\n'.format(epoch, max_epoch - 1), '-' * 30)\n\n            for phase in [\"train\", \"val\"]:\n\n                y_true = []\n                y_pred = []\n\n                if phase == \"train\":\n                    net.train()\n                else:\n                    net.eval()\n\n                running_loss = 0.0\n                running_corrects = 0\n\n                # Iterate data\n                for iteration, (x, y) in tqdm(enumerate(data_loader[phase])):\n\n                    x = x.to(DEVICE)  # FloatType\n                    y = y.to(DEVICE).float()  # LongType → FloatType\n                    optimizer.zero_grad()\n\n                    with torch.set_grad_enabled(phase == 'train'):\n                        outputs = net(x)  # outputs shape & y shape [64, 1]\n                        loss = criterion(outputs, y)\n\n                        proba = nn.Sigmoid()(outputs)\n                        preds = torch.round(proba)\n\n                        y_true.append(y.data.detach().cpu().numpy())\n                        y_pred.append(proba.detach().cpu().numpy())\n\n                        if phase == 'train':\n                            # backward + optimizer\n                            loss.backward()\n                            optimizer.step()\n\n                    running_loss += loss.item()  # get loss number\n                    running_corrects += torch.sum(preds == y.data)\n\n                y_true = np.vstack(y_true).reshape(-1)\n                y_pred = np.vstack(y_pred).reshape(-1)\n\n                if phase == \"train\":\n                    epoch_auc = roc_auc_score(y_true, y_pred)\n                    epoch_loss = running_loss / KF_length[idx][\"train\"]\n                    epoch_acc = running_corrects.double() / KF_length[idx][\"train\"]\n\n                    scheduler.step(epoch_acc)\n                else:\n                    epoch_auc = roc_auc_score(y_true, y_pred)\n                    epoch_loss = running_loss / KF_length[idx][\"val\"]\n                    epoch_acc = running_corrects.double() / KF_length[idx][\"val\"]\n\n                print('{} Loss: {:.4f} Acc: {:.4f}, Auc: {:.4f}'.format(\n                    phase, epoch_loss, epoch_acc, epoch_auc))\n\n                if phase == 'val' and epoch_acc > best_acc:\n                    best_acc = epoch_acc\n                    best_epoch = epoch\n                    best_model_wts = copy.deepcopy(net.state_dict())\n                if phase == 'val':\n                    print(\"Now epoch {} is the best epoch\".format(best_epoch))\n\n        net.load_state_dict(best_model_wts)\n        torch.save(net.state_dict(), \"{}_cv{}_best_epoch{}.pth\".format(net._get_name(), idx, best_epoch))\n        print(\"-\" * 30, \"\\nFinish Training cv{}, {} Epoch\\nThe Best is epoch {}\\n\".format(idx, MAX_EPOCH, best_epoch))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef predict():\n    for idx in range(5):\n        net = SE_DenseNet169_plus()\n        net.to(\"cpu\")\n\n        preds = []\n        weight_path = glob.glob(\"SE_DenseNet169_plus_cv{}*pth\".format(idx))\n        if len(weight_path) != 0:\n            print(\"-\" * 30, \"\\nLoading weight\")\n            print(\"Test data size: {}\\nTest data batch: {}\\n\".format(\n                len(test_data), len(test_data_loader)))\n            net.load_state_dict(torch.load(weight_path[0]))\n        else:\n            raise FileExistsError(\"Not exist weight file!\")\n\n        # batch_size, n_crops, c, h, w = data.size()\n        # data = data.view(-1, c, h, w)\n        # output = model(data)\n        # output = output.view(batch_size, n_crops, -1).mean(1)\n\n        net.to(DEVICE)\n        net.eval()\n        print(\"Start testing cv-{}\".format(idx))\n\n        with torch.no_grad():\n\n            for batch_i, (x_test, target) in tqdm(enumerate(test_data_loader)):\n                test_batch_size, n_crops, c, h, w = x_test.size()\n                x_test = x_test.view(-1, c, h, w)\n                x_test = x_test.to(DEVICE)\n                out = net(x_test)\n\n                batch_pred = nn.Sigmoid()(out)\n                batch_pred = batch_pred.view(test_batch_size, n_crops, -1).mean(1)\n\n                batch_pred = list(batch_pred.detach().cpu().numpy())\n\n                preds.append(batch_pred)\n\n        test_pred = pd.DataFrame({\"imgs\": test_data.image_data,\n                                  \"preds\": np.vstack(preds).reshape(-1)})\n        test_pred[\"imgs\"] = test_pred[\"imgs\"].apply(lambda x: x.split(\"/\")[-1][:-4])\n\n        sub = pd.merge(TEST_CSV, test_pred, left_on=\"id\", right_on=\"imgs\")\n        sub = sub[['id', 'preds']]\n        sub.columns = ['id', 'label']\n\n        if os.path.exists(\"./output\"):\n            sub.to_csv(\"./output/{}_{}_cv{}_TTA_proba.csv\".format(time.strftime(\"%m%d\"), net._get_name(), idx))\n        else:\n            os.mkdir(\"./output\")\n            sub.to_csv(\"./output/{}_{}_cv{}_TTA_proba.csv\".format(time.strftime(\"%m%d\"), net._get_name(), idx))\n        print(\"File Saved!\")\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}