{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install \"../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n!pip install \"../input/hpapytorchzoozip/pytorch_zoo-master\"\n!pip install \"../input/hpacellsegmentatormaster/HPA-Cell-Segmentation-master\"\nNUC_MODEL = '../input/hpacellsegmentatormodelweights/dpn_unet_nuclei_v1.pth'\nCELL_MODEL = '../input/hpacellsegmentatormodelweights/dpn_unet_cell_3ch_v1.pth'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport os\nimport re\n\nimport base64\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nimport cv2\nfrom tqdm import tqdm_notebook\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport hpacellseg.cellsegmentator as cellsegmentator\nfrom hpacellseg.utils import label_cell, label_nuclei\n\n# Build the Segmentator\nsegmentator = cellsegmentator.CellSegmentator(\n    NUC_MODEL,\n    CELL_MODEL,\n    scale_factor=0.25,\n    device='cuda',\n    padding=True,\n    multi_channel_model=True\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_binary_mask(mask: np.ndarray) -> t.Text:\n  \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n\n  # check input mask --\n  if mask.dtype != np.bool:\n    raise ValueError(\n        \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n        mask.dtype)\n\n  mask = np.squeeze(mask)\n  if len(mask.shape) != 2:\n    raise ValueError(\n        \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n        mask.shape)\n\n  # convert input mask to expected COCO API input --\n  mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n  mask_to_encode = mask_to_encode.astype(np.uint8)\n  mask_to_encode = np.asfortranarray(mask_to_encode)\n\n  # RLE encode mask --\n  encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n  # compress and base64 encoding --\n  binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n  base64_str = base64.b64encode(binary_str)\n  return base64_str.decode('utf-8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def segmentate(segmentator, \n               image_id = '000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0', \n               work='train'):\n    '''\n    Input:\n    segmentator: a implemented segmentator for the segmentation of image;\n    image_id <str>: the id of image;\n    work <str>: ['train'|'test'], get image from train or test directory;\n    _____________________________________________________________________\n    Output:\n    cell_mask_list <numpy tensor>: [n_mask, x, y];\n    encode_mask_list <list of str>: Encoded mask following the formula of kaggle submission;\n    '''\n    \n    mt = f'/kaggle/input/hpa-single-cell-image-classification/{work}/{image_id}_red.png'\n    er = f'/kaggle/input/hpa-single-cell-image-classification/{work}/{image_id}_yellow.png'\n    nu = f'/kaggle/input/hpa-single-cell-image-classification/{work}/{image_id}_blue.png'\n    pr = f'/kaggle/input/hpa-single-cell-image-classification/{work}/{image_id}_green.png'\n    \n    images = [[mt], [er], [nu]]\n\n    img_h = cv2.imread(mt).shape[0]\n    \n    cell_segmentations = segmentator.pred_cells(images)\n    nuc_segmentations = segmentator.pred_nuclei(images[2])\n    cell_nuclei_mask, cell_mask = label_cell(nuc_segmentations[0], cell_segmentations[0])\n\n    numbers = set(np.ravel(cell_mask))\n    numbers.remove(0)\n\n    index = 1\n\n    cell_mask_list = np.array([])\n    encode_mask_list = np.array([])\n\n    for number in numbers:\n        isolated_cell = np.where(cell_mask==number, 1, 0)\n        mask_to_bool = isolated_cell.astype(bool)\n        encode_mask = encode_binary_mask(mask_to_bool)\n        index += 1\n\n        isolated_cell = np.expand_dims(isolated_cell, 0)\n        encode_mask = np.expand_dims(encode_mask, 0)\n        \n        if number == 1: \n            cell_mask_list = np.array(isolated_cell)\n            encode_mask_list = np.array(encode_mask)\n        else: \n            cell_mask_list = np.concatenate((cell_mask_list, isolated_cell))\n            encode_mask_list = np.concatenate((encode_mask_list, encode_mask))\n            \n        if img_h > 2048 and index == 32:\n            break\n            \n    return cell_mask_list, encode_mask_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def HPA_image(image_id='5c27f04c-bb99-11e8-b2b9-ac1f6b6435d0', work='train'):\n    '''\n    get a image of shape (Height, Width, 12*) from the kaggle dataset\n    * 12 from 3*4 concatenated rgb channels from image of red, yellow, blue and green\n    '''\n    mt = f'/kaggle/input/hpa-single-cell-image-classification/{work}/{image_id}_red.png'\n    er = f'/kaggle/input/hpa-single-cell-image-classification/{work}/{image_id}_yellow.png'\n    nu = f'/kaggle/input/hpa-single-cell-image-classification/{work}/{image_id}_blue.png'\n    pr = f'/kaggle/input/hpa-single-cell-image-classification/{work}/{image_id}_green.png'\n\n    images = [cv2.imread(mt), cv2.imread(er), cv2.imread(nu), cv2.imread(pr)]\n    image = np.concatenate(images, -1)\n    return image\n\ndef make_bound(mask):\n    '''\n    make bounds for masks\n       y_l____y_u\n    x_l         |\n    |           |\n    |    mask   |\n    x_u         |\n    |___________|\n    '''\n    x_edge, y_edge = (np.sum(mask, 1) > 0), (np.sum(mask, 0) > 0)\n    x_l, y_l = np.argmax(x_edge), np.argmax(y_edge)\n    x_u, y_u = len(x_edge)-1-np.argmax(x_edge[::-1]), len(y_edge)-1-np.argmax(y_edge[::-1])\n    return (x_l, x_u), (y_l, y_u)\n\ndef image_cut(image, mask):\n    '''\n    cut off mask's 0's background\n    '''\n    (x_u, x_d), (y_u, y_d) = make_bound(mask)\n    image = image * np.expand_dims(mask, -1)\n    return image[x_u:x_d, y_u:y_d, :]\n\ndef image_seize(image, mask, dsize=(64, 64)):\n    image = cv2.resize(image_cut(image, mask).astype('float32'), dsize=dsize)\n    return image\n\ndef image_parse(segmentator, \n               image_id = '000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0', \n               work='train',\n               dsize=(64, 64)):\n    \n    cell_mask_list, encode_mask_list = segmentate(segmentator, \n               image_id = image_id, \n               work=work)\n    \n    image = HPA_image(image_id, work)\n    \n    cell_images = [np.expand_dims(image_seize(image, mask, dsize), 0) for mask in cell_mask_list]\n    \n    cell_batch = np.concatenate(cell_images, 0)\n    \n    return cell_batch, encode_mask_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def batch_generate(ran=None, dsize=(64, 64)):\n    df = pd.read_csv('/kaggle/input/hpa-single-cell-image-classification/train.csv')\n    df_one_label = df[df.Label.apply(lambda seq: len(seq.split('|'))==1)]\n    n_image = len(df_one_label)\n\n    if ran is not None:\n        df_slice = df_one_label.iloc[ran[0]%n_image:ran[0]%n_image+(ran[1]-ran[0])]\n    else:\n        df_slice = df_one_label\n\n    cell_batches, label_batches = [], []\n\n    for _, row in df_slice.iterrows():\n        image_id, label = row['ID'], row['Label']\n        cell_batch, _ = image_parse(segmentator,\n                                       image_id = image_id, \n                                       work='train',\n                                       dsize=dsize)\n        label_batch = np.array([int(label) for _ in range(cell_batch.shape[0])])\n\n        cell_batches.append(cell_batch)\n        label_batches.append(label_batch)\n\n    cells, labels = np.concatenate(cell_batches, 0), np.concatenate(label_batches, 0)\n    \n    return cells, labels\n\ndef minibatch_generate(cells, labels, batch_size=16):\n    ids = np.random.permutation(np.arange(cells.shape[0]))\n    cells, labels = cells[ids, :, :, :], labels[ids]\n    idx = 0\n    while idx + batch_size < cells.shape[0]:\n        yield cells[idx:idx+batch_size, :, :, :], labels[idx:idx+batch_size]\n        idx += batch_size\n    yield cells[idx:, :, :, :], labels[idx:]\n    \ndef unified_generate(ran=None, dsize=(64, 64), batch_size=16):\n    cells, labels = batch_generate(ran=ran, dsize=dsize)\n    for cells_mini, labels_mini in minibatch_generate(cells, labels):\n        yield cells_mini, labels_mini","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def batch_generate_test(ran=None, dsize=(64, 64)):\n    os.chdir(r\"/kaggle/input/hpa-single-cell-image-classification/test/\")\n    filenames = glob.glob(\"**_red.png\")  \n    image_ids = [re.findall(\"(.*)_red.png\", filename)[0] for filename in filenames]\n    n_image = len(image_ids)\n    \n    if ran is not None:\n        image_ids = image_ids[ran[0]%n_image:ran[1]%n_image]\n\n    for image_id in image_ids:\n        cell_batch, encode_mask_list = image_parse(segmentator,\n                                       image_id = image_id, \n                                       work='test',\n                                       dsize=dsize)\n        \n        image = HPA_image(image_id=image_id, work='test')\n        \n        width, height = image.shape[1], image.shape[0]\n\n        yield cell_batch, encode_mask_list, image_id, width, height\n        \ndef create_predstr(label, confidence, encode_mask_list):\n    strs = [' '.join([str(l), str(c), str(eml)]) for l, c, eml in zip(label, confidence, encode_mask_list)]\n    predstr = ' '.join(strs)\n    return predstr","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN2d(nn.Module):\n    def __init__(self, device):\n        super(CNN2d, self).__init__()\n        self.conv0 = nn.Conv2d(12, 16, (5, 5), padding=2)\n        self.pool0 = nn.AvgPool2d((2, 2), 2)\n        self.conv1 = nn.Conv2d(16, 32, (5, 5), padding=2)\n        self.pool1 = nn.AvgPool2d((2, 2), 2)\n        self.conv2 = nn.Conv2d(32, 64, (5, 5), padding=2)\n        self.pool2 = nn.MaxPool2d((2, 2), 2)\n        self.fc = nn.Sequential(\n            nn.Linear(4096, 19),\n        )\n        \n        self.criterion = nn.CrossEntropyLoss()\n    \n        self.optimizer = optim.Adam(self.parameters(), lr=1e-5)\n        \n        self.device = device\n        \n        self.train_buffer = None\n        self.valid_buffer = None\n        \n        self.buffer_train_max = 512\n        self.buffer_valid_max = 128\n        \n        self.acc_threshold = 0.33\n        \n        self.dead_lim = 5\n        \n        self.dead_cnt = self.dead_lim\n        \n    def forward(self, x):\n        x = self.conv0(x)\n        x = self.pool0(x)\n        x = self.conv1(x)\n        x = self.pool1(x)\n        x = self.conv2(x)\n        x = self.pool2(x)\n        x = x.reshape(x.shape[0], -1)\n        x = self.fc(x)\n        return x\n    \n    def loss(self, pred, y):\n        return self.criterion(pred, y)\n    \n    def fit(self, x, y):\n        self.zero_grad()\n        pred = self(x)\n        loss = self.loss(pred, y)\n        loss.backward()\n        self.optimizer.step()\n        loss = loss.cpu().detach().numpy()\n        return loss\n        \n    def buffer_train_load(self, cells_mini, labels_mini):\n        if self.train_buffer is None:\n            self.train_buffer = (cells_mini, labels_mini)\n        else:\n            self.train_buffer = (torch.cat([self.train_buffer[0], cells_mini], 0), \\\n                                 torch.cat([self.train_buffer[1], labels_mini], 0))\n        \n        self.train_buffer = (self.train_buffer[0][torch.randperm(self.train_buffer[0].shape[0])], \\\n                             self.train_buffer[1][torch.randperm(self.train_buffer[1].shape[0])])\n        \n        if self.train_buffer[0].shape[0] > self.buffer_train_max:\n            self.train_buffer = (self.train_buffer[0][:self.buffer_train_max], \\\n                                 self.train_buffer[1][:self.buffer_train_max])\n        \n    def buffer_valid_load(self, cells_mini, labels_mini):\n        if self.valid_buffer is None:\n            self.valid_buffer = (cells_mini, labels_mini)\n        else:\n            self.valid_buffer = (torch.cat([self.valid_buffer[0], cells_mini], 0), \\\n                                 torch.cat([self.valid_buffer[1], labels_mini], 0))\n        \n        self.valid_buffer = (self.valid_buffer[0][torch.randperm(self.valid_buffer[0].shape[0])], \\\n                             self.valid_buffer[1][torch.randperm(self.valid_buffer[1].shape[0])])\n        \n        if self.valid_buffer[0].shape[0] > self.buffer_valid_max:\n            self.valid_buffer = (self.valid_buffer[0][:self.buffer_valid_max], \\\n                                 self.valid_buffer[1][:self.buffer_valid_max])\n            \n    def buffer_generator(self, src='train', batch_size=16):\n        if src == 'train':\n            buffer = self.train_buffer\n        elif src == 'valid':\n            buffer = self.valid_buffer\n            \n        idx = 0\n        while idx * batch_size < buffer[0].shape[0]:\n            yield (buffer[0][idx*batch_size:(idx+1)*batch_size], \\\n                   buffer[1][idx*batch_size:(idx+1)*batch_size])\n            idx += 1\n    \n    def train(self, n_epoch=100, e_valid=8, ran_stride=1, dsize=(64, 64), batch_size=16):\n        for epoch in range(n_epoch):\n            log = \"\"\n            log += f'Epoch#{epoch};'\n            train_losses = []\n            idx = np.random.randint(1e8)\n            unified_generator = unified_generate(ran=(idx, idx + ran_stride), dsize=dsize, batch_size=batch_size)\n            for cells_mini, labels_mini in unified_generator:\n                cells_mini, labels_mini = torch.FloatTensor(cells_mini).to(self.device), torch.LongTensor(labels_mini).to(self.device)\n#                 plt.imshow(cells_mini[0, :, :, 0])\n#                 plt.show()\n                cells_mini = cells_mini.permute(0, 3, 1, 2)\n#                 loss = self.fit(cells_mini, labels_mini)\n#                 train_losses.append(loss)\n                self.buffer_train_load(cells_mini, labels_mini)\n                \n            if epoch > 10:\n                for cells_mini, labels_mini in self.buffer_generator(batch_size=batch_size, src='train'):\n    #                 cells_mini, labels_mini = torch.FloatTensor(cells_mini).to(self.device), torch.LongTensor(labels_mini).to(self.device)\n                    loss = self.fit(cells_mini, labels_mini)\n                    train_losses.append(loss)\n                \n                train_loss = sum(train_losses)/len(train_losses)\n                log += f'Train Loss#{train_loss};'\n\n                if (epoch+1) % e_valid == 0:\n                    acc = self.valid(batch_size=batch_size)\n                    log += f'Accuracy#{acc};'\n                    if acc > self.acc_threshold:\n                        self.dead_cnt -= 1\n                    else:\n                        self.dead_cnt = self.dead_lim\n\n                    if self.dead_cnt == 0:\n                        break\n            print(log)\n                \n    def valid(self, ran_stride=1, dsize=(64, 64), batch_size=16):\n        results = []\n        idx = np.random.randint(1e8)\n        unified_generator = unified_generate(ran=(idx, idx + ran_stride), dsize=dsize, batch_size=batch_size)\n        for cells_mini, labels_mini in unified_generator:\n            cells_mini, labels_mini = torch.FloatTensor(cells_mini).to(self.device), torch.LongTensor(labels_mini).to(self.device)\n            cells_mini = cells_mini.permute(0, 3, 1, 2)\n            self.buffer_valid_load(cells_mini, labels_mini)\n        \n        for cells_mini, labels_mini in self.buffer_generator(batch_size=batch_size, src='valid'):\n            preds_mini = self(cells_mini)\n            preds_mini = torch.argmax(preds_mini, -1)\n            preds_mini, labels_mini = preds_mini.cpu().detach().numpy(), labels_mini.cpu().detach().numpy()\n            result = np.equal(preds_mini, labels_mini)\n            results.append(result)\n        results = [item for result in results for item in result]\n        acc = sum(results)/len(results)\n        return acc\n        \n    def pred(self, cell_batch):\n        pred = self(cell_batch)\n        pred = torch.softmax(pred, -1)\n        label = torch.argmax(pred, -1)\n        confidence, _ = torch.max(pred, -1)\n        return label, confidence\n    \n    def test(self):\n        ID, ImageWidth, ImageHeight, PredictionString = [], [], [], []\n        for cell_batch, encode_mask_list, image_id, width, height in batch_generate_test(ran=None, dsize=(64, 64)):\n            cell_batch = torch.FloatTensor(cell_batch).to(device)\n            cell_batch = cell_batch.permute(0, 3, 1, 2)\n            label, confidence = self.pred(cell_batch)\n            label, confidence = label.cpu().detach().numpy(), confidence.cpu().detach().numpy()\n            predstr = create_predstr(label, confidence, encode_mask_list)\n            ID.append(image_id)\n            ImageWidth.append(width)\n            ImageHeight.append(height)\n            PredictionString.append(predstr)\n#             break\n\n        df = pd.DataFrame({\n            'ID':ID,\n            'ImageWidth':ImageWidth, \n            'ImageHeight':ImageHeight, \n            'PredictionString':PredictionString,\n        })\n        return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision\nclass ResNet(CNN2d): # <--Build your ResNet here!\n    def __init__(self, device):\n        super().__init__(device)\n        self.model = torchvision.models.resnet50(pretrained=True)\n        self.model.conv1 = nn.Conv2d(12, 64, kernel_size=7, stride=1, padding=3, bias=False)\n        #提取fc层中固定的参数\n        self.fc_features = self.model.fc.in_features\n        self.OUT_CLASSES = 19\n        #修改类别数目为OUT_CLASSES\n        self.model.fc = nn.Linear(self.fc_features, self.OUT_CLASSES)\n        \n        self.criterion = nn.CrossEntropyLoss()\n    \n        self.optimizer = optim.Adam(self.parameters(), lr=1e-4)\n        \n        self.device = device\n        \n        self.train_buffer = None\n        self.valid_buffer = None\n        \n        self.buffer_train_max = 2048\n        self.buffer_valid_max = 512\n        \n        self.acc_threshold = 0.6\n        \n        self.dead_lim = 5\n        \n        self.dead_cnt = self.dead_lim\n        \n    def forward(self, x):\n        return self.model(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dsize = (64, 64)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = ResNet(device).to(device)\nmodel.train(n_epoch=100, batch_size=64) # About 45min(CPU) for one train epoch (1min30s with valid)\ndf_submit = model.test() # About 40s(CPU) for one image (Total: About 5~7h)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit # <--The file for submission\ndf_submit.to_csv(\"/kaggle/working/submission.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}