{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **KDDM 2 SARTORIUS CELL INSTANCE SEGMENTATION U-NET TRAIN AND INFERENCE NOTEBOOK**","metadata":{}},{"cell_type":"markdown","source":"# INSTALL All DEPENDENCIES\n* **Install segmentation_models.pytorch and required dependencies**\n* **Import libaries**\n* **Helper function to visualize images**","metadata":{}},{"cell_type":"code","source":"!cp -r /kaggle/input/kddm2/unet /kaggle/working/carrotssmp\n\n!pip install /kaggle/working/carrotssmp/timm-0.4.12/timm-0.4.12-py3-none-any.whl --no-deps\n!cd /kaggle/working/carrotssmp/efficientnet_pytorch-0.6.3/ && pip install -e .\n!cd /kaggle/working/carrotssmp/pretrainedmodels-0.7.4/ && pip install -e .\n!pip install /kaggle/working/carrotssmp/segmentation_models.pytorch-0.2.1/segmentation_models_pytorch-0.2.1-py3-none-any.whl --no-deps\n\nimport sys\nsys.path.append(\"/kaggle/working/carrotssmp/efficientnet_pytorch-0.6.3/\")\nsys.path.append(\"/kaggle/working/carrotssmp/pretrainedmodels-0.7.4/\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-26T11:15:48.717058Z","iopub.execute_input":"2022-01-26T11:15:48.717448Z","iopub.status.idle":"2022-01-26T11:17:45.63225Z","shell.execute_reply.started":"2022-01-26T11:15:48.717355Z","shell.execute_reply":"2022-01-26T11:17:45.631206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cupy as cp\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nfrom tqdm import tqdm\n\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset as BaseDataset\n\nimport albumentations as albu\n\nfrom scipy.ndimage.measurements import label\nimport segmentation_models_pytorch as smp\nimport shutil\nimport random\nimport torch","metadata":{"execution":{"iopub.status.busy":"2022-01-26T11:19:26.86822Z","iopub.execute_input":"2022-01-26T11:19:26.868578Z","iopub.status.idle":"2022-01-26T11:19:35.21795Z","shell.execute_reply.started":"2022-01-26T11:19:26.868542Z","shell.execute_reply":"2022-01-26T11:19:35.216736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# helper function for data visualization\ndef visualize(**images):\n    \"\"\"PLot images in one row.\"\"\"\n    n = len(images)\n    plt.figure(figsize=(16, 5))\n    for i, (name, image) in enumerate(images.items()):\n        plt.subplot(1, n, i + 1)\n        plt.xticks([])\n        plt.yticks([])\n        plt.title(' '.join(name.split('_')).title())\n        plt.imshow(image)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-26T11:19:35.220873Z","iopub.execute_input":"2022-01-26T11:19:35.221227Z","iopub.status.idle":"2022-01-26T11:19:35.229822Z","shell.execute_reply.started":"2022-01-26T11:19:35.221175Z","shell.execute_reply":"2022-01-26T11:19:35.228168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATALOADER\n* **Read annotation csv and generate mask immages**\n","metadata":{}},{"cell_type":"code","source":"SARTORIUS_CLASSES = ['astro', 'cort', 'shsy5y']\n\n#define path to input image folder, annotation csv and output folders for images and masks\nimg_dir = \"/kaggle/input/sartorius-cell-instance-segmentation/train\"\nannotation_file = \"/kaggle/input/sartorius-cell-instance-segmentation/train.csv\"\nx_train_dir = \"/kaggle/working/train\"\ny_train_dir = \"/kaggle/working/trainannot\"\nx_val_dir = \"/kaggle/working/val\"\ny_val_dir = \"/kaggle/working/valannot\"\n\nos.makedirs(x_train_dir, exist_ok=True)\nos.makedirs(y_train_dir, exist_ok=True)\nos.makedirs(x_val_dir, exist_ok=True)\nos.makedirs(y_val_dir, exist_ok=True)\n\n#random train/val split\npercentage_in_train = 0.9\nframes = os.listdir(img_dir)\nnum_train = int(len(frames) * percentage_in_train)\nrandom.seed(0)\ntrain_frames = set( random.sample(frames, num_train) )\n\nwith open(annotation_file, \"r\") as f:\n    anns = f.read().split(\"\\n\")\nanns = anns[1:-1]\nprint(\"annotations\", len(anns))\n\n#create dict with annotation masks per img and convert rle to \"start to end\" pixel masks\nanns_per_img_id = {}\nfor a in anns:\n    values = a.split(\",\")\n    id = values[0]\n    cell_type = values[4]\n    rle_mask = values[1].split(\" \")\n\n    starts = list(map(lambda x: int(x) - 1, rle_mask[0::2]))\n    lengths = list(map(int, rle_mask[1::2]))\n    ends = [x + y for x, y in zip(starts, lengths)]\n\n    if id not in anns_per_img_id.keys():\n        anns_per_img_id[id] = {}\n        anns_per_img_id[id][\"start_end\"] = []\n\n    anns_per_img_id[id][\"start_end\"].append( [starts, ends] )\n    anns_per_img_id[id][\"cell_type\"] = cell_type\nnum_imgs_keys = len(anns_per_img_id.keys())\nprint(\"anns_per_img_id\", num_imgs_keys)\n\n\n#generate image masks\npixel_val_per_class = {'astro': 1, 'cort': 2, 'shsy5y': 3}\nimg_mask = np.empty((704*520), dtype=np.uint8)\nimg_mask_2d = np.empty((520, 704), dtype=np.uint8)\nfor i in tqdm( range(num_imgs_keys) ):\n    k = list(anns_per_img_id.keys())[i]\n    anns = anns_per_img_id[k]\n    masks = anns[\"start_end\"]\n    id = anns[\"cell_type\"]\n    img_mask.fill(0)\n    for m in masks:\n        for start, end in zip(m[0], m[1]):\n            img_mask[start:end] = pixel_val_per_class[id]\n    img_mask_2d = img_mask.reshape((520, 704))\n    fname = \"{:s}.png\".format(k)\n    if fname in train_frames:\n        shutil.copy( os.path.join(img_dir, fname), os.path.join(x_train_dir, fname) )\n        cv2.imwrite( os.path.join(y_train_dir, fname), img_mask_2d )\n    else:\n        shutil.copy( os.path.join(img_dir, fname), os.path.join(x_val_dir, fname) )\n        cv2.imwrite( os.path.join(y_val_dir, fname), img_mask_2d )\n    \nprint( \"train imgs:\", len(os.listdir(x_train_dir)), \"train mask imgs:\", len(os.listdir(y_train_dir)) )\nprint( \"val imgs:\", len(os.listdir(x_val_dir)), \"val mask imgs:\", len(os.listdir(y_val_dir)) )","metadata":{"execution":{"iopub.status.busy":"2022-01-26T11:25:22.00219Z","iopub.execute_input":"2022-01-26T11:25:22.002478Z","iopub.status.idle":"2022-01-26T11:25:34.019711Z","shell.execute_reply.started":"2022-01-26T11:25:22.002446Z","shell.execute_reply":"2022-01-26T11:25:34.018654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **Class for loading the dataset and applying augmentations/preprocessing**","metadata":{}},{"cell_type":"code","source":"class Dataset(BaseDataset):\n    \"\"\"Read images, apply augmentation and preprocessing transformations.\n    \n    Args:\n        images_dir (str): path to images folder\n        masks_dir (str): path to segmentation masks folder\n        class_values (list): values of classes to extract from segmentation mask\n        augmentation (albumentations.Compose): data transfromation pipeline \n            (e.g. flip, scale, etc.)\n        preprocessing (albumentations.Compose): data preprocessing \n            (e.g. noralization, shape manipulation, etc.)\n    \n    \"\"\"\n    \n    CLASSES = SARTORIUS_CLASSES\n    \n    def __init__(\n            self, \n            images_dir, \n            masks_dir, \n            classes=None, \n            augmentation=None, \n            preprocessing=None,\n    ):\n        self.ids = os.listdir(images_dir)\n        self.images_fps = [os.path.join(images_dir, image_id) for image_id in self.ids]\n        self.masks_fps = [os.path.join(masks_dir, image_id) for image_id in self.ids]\n        \n        # convert str names to class values on masks\n        self.class_values = [self.CLASSES.index(cls.lower()) for cls in classes]\n        \n        self.augmentation = augmentation\n        self.preprocessing = preprocessing\n    \n    def __getitem__(self, i):\n        \n        # read data\n        image = cv2.imread(self.images_fps[i])\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        mask = cv2.imread(self.masks_fps[i], 0)\n        \n        # extract certain classes from mask (e.g. cars)\n        masks = [(mask == v) for v in self.class_values]\n        mask = np.stack(masks, axis=-1).astype('float')\n        \n        # apply augmentations\n        if self.augmentation:\n            sample = self.augmentation(image=image, mask=mask)\n            image, mask = sample['image'], sample['mask']\n\n        # apply preprocessing\n        if self.preprocessing:\n            sample = self.preprocessing(image=image, mask=mask)\n            image, mask = sample['image'], sample['mask']\n\n        return image, mask\n        \n    def __len__(self):\n        return len(self.ids)\n    \n    def getItemPerFname(self, fname):\n        # read data\n        image = cv2.imread(fname)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        mask = np.zeros((image.shape[0], image.shape[1]))\n        \n        # extract certain classes from mask (e.g. cars)\n        masks = [(mask == v) for v in self.class_values]\n        mask = np.stack(masks, axis=-1).astype('float')\n        \n        # apply augmentations\n        if self.augmentation:\n            sample = self.augmentation(image=image, mask=mask)\n            image, mask = sample['image'], sample['mask']\n\n        # apply preprocessing\n        if self.preprocessing:\n            sample = self.preprocessing(image=image, mask=mask)\n            image, mask = sample['image'], sample['mask']\n\n        return image, mask\n        ","metadata":{"execution":{"iopub.status.busy":"2022-01-26T11:25:57.667446Z","iopub.execute_input":"2022-01-26T11:25:57.667732Z","iopub.status.idle":"2022-01-26T11:25:57.686972Z","shell.execute_reply.started":"2022-01-26T11:25:57.6677Z","shell.execute_reply":"2022-01-26T11:25:57.685961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **Load train dataset**","metadata":{}},{"cell_type":"code","source":"dataset = Dataset(x_train_dir, y_train_dir, classes=SARTORIUS_CLASSES)\n\nimage, mask = dataset[5] # get some sample\nvisualize(\n    image=image, \n    cell_masks=mask.squeeze(),\n)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T11:26:00.239466Z","iopub.execute_input":"2022-01-26T11:26:00.239765Z","iopub.status.idle":"2022-01-26T11:26:00.580497Z","shell.execute_reply.started":"2022-01-26T11:26:00.239733Z","shell.execute_reply":"2022-01-26T11:26:00.579458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **Define augmentations**","metadata":{}},{"cell_type":"code","source":"def get_training_augmentation():\n    train_transform = [\n        albu.HorizontalFlip(p=0.5),\n        albu.ShiftScaleRotate(scale_limit=0.5, rotate_limit=0, shift_limit=0.1, p=1, border_mode=0),\n        albu.PadIfNeeded(min_height=320, min_width=320, always_apply=True, border_mode=0),\n        albu.RandomCrop(height=320, width=320, always_apply=True),\n        albu.IAAAdditiveGaussianNoise(p=0.2),\n        albu.IAAPerspective(p=0.5),\n\n        albu.OneOf(\n            [\n                albu.CLAHE(p=1),\n                albu.RandomBrightness(p=1),\n                albu.RandomGamma(p=1),\n            ],\n            p=0.9,\n        ),\n\n        albu.OneOf(\n            [\n                albu.IAASharpen(p=1),\n                albu.Blur(blur_limit=3, p=1),\n                albu.MotionBlur(blur_limit=3, p=1),\n            ],\n            p=0.9,\n        ),\n\n        albu.OneOf(\n            [\n                albu.RandomContrast(p=1),\n                albu.HueSaturationValue(p=1),\n            ],\n            p=0.9,\n        ),\n    ]\n    return albu.Compose(train_transform)\n\n\ndef get_validation_augmentation():\n    \"\"\"Add paddings to make image shape divisible by 32\"\"\"\n    test_transform = [\n        albu.PadIfNeeded(544, 702)\n    ]\n    return albu.Compose(test_transform)\n\n\ndef to_tensor(x, **kwargs):\n    return x.transpose(2, 0, 1).astype('float32')\n\n\ndef get_preprocessing(preprocessing_fn):\n    \"\"\"Construct preprocessing transform\n    \n    Args:\n        preprocessing_fn (callbale): data normalization function \n            (can be specific for each pretrained neural network)\n    Return:\n        transform: albumentations.Compose\n    \n    \"\"\"\n    \n    _transform = [\n        albu.Lambda(image=preprocessing_fn),\n        albu.Lambda(image=to_tensor, mask=to_tensor),\n    ]\n    return albu.Compose(_transform)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T11:26:03.32115Z","iopub.execute_input":"2022-01-26T11:26:03.32145Z","iopub.status.idle":"2022-01-26T11:26:03.334844Z","shell.execute_reply.started":"2022-01-26T11:26:03.321417Z","shell.execute_reply":"2022-01-26T11:26:03.333551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **TRAINING**\n* **Set settings for training**\n* **Execute training**","metadata":{}},{"cell_type":"code","source":"!mkdir -p /root/.cache/torch/hub/checkpoints/\n!cp /kaggle/input/kddm2/unet/resnext50_32x4d-7cdf4587.pth /root/.cache/torch/hub/checkpoints/resnext50_32x4d-7cdf4587.pth\n!ls /root/.cache/torch/hub/checkpoints/","metadata":{"execution":{"iopub.status.busy":"2022-01-26T11:26:18.760968Z","iopub.execute_input":"2022-01-26T11:26:18.761988Z","iopub.status.idle":"2022-01-26T11:26:21.509689Z","shell.execute_reply.started":"2022-01-26T11:26:18.761947Z","shell.execute_reply":"2022-01-26T11:26:21.508574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **Define model, activation function etc. and initialize epoch runners**","metadata":{}},{"cell_type":"code","source":"ENCODER = 'resnext50_32x4d'\nENCODER_WEIGHTS = 'imagenet'\nACTIVATION = 'sigmoid' # could be None for logits or 'softmax2d' for multiclass segmentation\nDEVICE = 'cuda'\n\n\n# create segmentation model with pretrained encoder\nmodel = smp.FPN(\n    encoder_name=ENCODER, \n    encoder_weights=ENCODER_WEIGHTS, \n    classes=len(SARTORIUS_CLASSES), \n    activation=ACTIVATION,\n)\npreprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER, ENCODER_WEIGHTS)\n\n\n#define loss function and optimizer\nloss = smp.utils.losses.DiceLoss()\nmetrics = [\n    smp.utils.metrics.IoU(threshold=0.8),\n]\n\noptimizer = torch.optim.Adam([ \n    dict(params=model.parameters(), lr=0.0001),\n])\n\n\n#define datasets and data loader\ntrain_dataset = Dataset(\n    x_train_dir, \n    y_train_dir, \n    augmentation=get_training_augmentation(), \n    preprocessing=get_preprocessing(preprocessing_fn),\n    classes=SARTORIUS_CLASSES,\n)\n\nvalid_dataset = Dataset(\n    x_val_dir, \n    y_val_dir, \n    augmentation=get_validation_augmentation(), \n    preprocessing=get_preprocessing(preprocessing_fn),\n    classes=SARTORIUS_CLASSES,\n)\ntrain_loader = DataLoader(train_dataset, batch_size=3, shuffle=True, num_workers=2)\nvalid_loader = DataLoader(valid_dataset, batch_size=1, shuffle=False, num_workers=2)\n\n\n#create epoch runners (simple loop of iterating over dataloader`s samples)\ntrain_epoch = smp.utils.train.TrainEpoch(\n    model, \n    loss=loss, \n    metrics=metrics, \n    optimizer=optimizer,\n    device=DEVICE,\n    verbose=True,\n)\n\nvalid_epoch = smp.utils.train.ValidEpoch(\n    model, \n    loss=loss, \n    metrics=metrics, \n    device=DEVICE,\n    verbose=True,\n)","metadata":{"execution":{"iopub.status.busy":"2022-01-26T11:26:25.323916Z","iopub.execute_input":"2022-01-26T11:26:25.324541Z","iopub.status.idle":"2022-01-26T11:26:29.583351Z","shell.execute_reply.started":"2022-01-26T11:26:25.324501Z","shell.execute_reply":"2022-01-26T11:26:29.582281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* **Train model**","metadata":{}},{"cell_type":"code","source":"out_dir = \"/kaggle/working/results/\"\nos.makedirs(out_dir, exist_ok=True)\n\nmax_score = 0\nmax_epochs = 30\n# train model for \nfor i in range(0, max_epochs):\n    print('\\nEpoch: {}'.format(i+1))\n    train_logs = train_epoch.run(train_loader)\n    valid_logs = valid_epoch.run(valid_loader)\n    \n    # do something (save model, change lr, etc.)\n    if max_score < valid_logs['iou_score']: #todo\n        max_score = valid_logs['iou_score'] #todo\n        torch.save(model, out_dir + \"best_model.pth\")\n        print('Model saved!')\n        \n    if i == 25:\n        optimizer.param_groups[0]['lr'] = 1e-5\n        print('Decrease decoder learning rate to 1e-5!')","metadata":{"execution":{"iopub.status.busy":"2022-01-26T11:26:29.937408Z","iopub.execute_input":"2022-01-26T11:26:29.937699Z","iopub.status.idle":"2022-01-26T11:28:06.630759Z","shell.execute_reply.started":"2022-01-26T11:26:29.937666Z","shell.execute_reply":"2022-01-26T11:28:06.629621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **INFERENCE**\n* **Perform inference on the test dataset**\n* **Output sample images for visualization**\n* **Store in kaggle competition submission format**","metadata":{}},{"cell_type":"code","source":"def mask2rle(msk):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    msk    = cp.array(msk)\n    pixels = msk.flatten()\n    pad    = cp.array([0])\n    pixels = cp.concatenate([pad, pixels, pad])\n    runs   = cp.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\n\nx_test_dir = \"/kaggle/input/sartorius-cell-instance-segmentation/test\"\ny_test_dir = \"/kaggle/working/testannot\"\nos.makedirs(y_test_dir, exist_ok=True)\n\ndummy_test_mask = np.zeros((520, 704), dtype=np.uint8)\nfor f in os.listdir(x_test_dir):\n    cv2.imwrite( os.path.join(y_test_dir, f), dummy_test_mask )\n\ntest_dataset = Dataset(\n    x_test_dir, \n    y_test_dir, \n    augmentation=get_validation_augmentation(), \n    preprocessing=get_preprocessing(preprocessing_fn),\n    classes=SARTORIUS_CLASSES,\n)\n\ntest_dataset_vis = Dataset(\n    x_test_dir,\n    y_test_dir, \n    classes=SARTORIUS_CLASSES,\n)\n\nwith open(out_dir + \"best_model.pth\", \"rb\") as f:\n    best_model = torch.load(f)\n\n#run inference on test files and store submissions\noutput = {\"rles\": [], \"ids\": []}\ntest_files = sorted(os.listdir(x_test_dir))\nfor test_file in test_files:\n    #load file\n    id = test_file[:-4]\n    file_path = os.path.join(x_test_dir, test_file)\n    image_vis = test_dataset_vis.getItemPerFname(file_path)[0].astype('uint8')\n    image = test_dataset.getItemPerFname(file_path)[0]\n    \n    \n    #perform inference on image\n    x_tensor = torch.from_numpy(image).to(DEVICE)[None]\n    pr_mask = best_model.predict(x_tensor).detach()\n    pr_mask = (pr_mask.squeeze().cpu().numpy().round())\n    \n    \n    #visualize input image with mask overlay\n    merged = np.zeros((520, 704, 3), dtype=np.uint8)\n    for aa in range(merged.shape[0]):\n        for bb in range(merged.shape[1]):\n            if pr_mask[0, aa+12, bb] < 0.5:\n                merged[aa, bb, 0] = 255\n                merged[aa, bb, 1] = 5\n                merged[aa, bb, 2] = 0\n            else:\n                merged[aa, bb, 0] = image_vis[aa, bb, 0]\n                merged[aa, bb, 1] = image_vis[aa, bb, 1]\n                merged[aa, bb, 2] = image_vis[aa, bb, 2]\n    cv2.imwrite( os.path.join(\"/kaggle/working/\", \"{:s}_merged.png\".format(id)), merged )\n    \n    \n    #split mask into instance masks\n    uint8_mask = (255 - pr_mask[0]*255).astype(np.uint8)[12:-12, :]\n    labeled_colorized = np.zeros((520, 704, 3), dtype=np.uint8)\n    structure = np.ones((3, 3), dtype=np.int)\n    labeled, ncomponents = label(uint8_mask, structure)\n    \n    \n    #visualize input image with instance masks overlay\n    labels, counts = np.unique(labeled, return_counts=True)\n    bg_label = labels[np.argmax(counts)]\n    print(\"bg_label\", bg_label)\n    colors = np.random.choice(range(256), size=3*ncomponents)\n    for aa in range(labeled_colorized.shape[0]):\n        for bb in range(labeled_colorized.shape[1]):\n            if labeled[aa, bb] == bg_label:\n                continue\n            c = 3*(labeled[aa, bb]-1)\n            for i in range(3):\n                labeled_colorized[aa, bb, i] = colors[c+i]\n    plt.figure(figsize=(8, 8))\n    plt.imshow(labeled_colorized)\n    cv2.imwrite( os.path.join(\"/kaggle/working/\", \"{:s}_labeled.png\".format(id)), labeled_colorized )\n    \n    \n    #convert predictions to kaggle submission format\n    for l in labels:\n        single_instance_mask = np.zeros((520, 704))\n        if l == bg_label:\n            continue\n        x_ind = np.where(labeled == l)[0]\n        y_ind = np.where(labeled == l)[1]\n        single_instance_mask[x_ind, y_ind] = 1\n        rle_mask =  mask2rle(single_instance_mask)\n        output[\"rles\"].append(rle_mask)\n        output[\"ids\"].append(id)\n\n#store predictions in kaggle submission format\nindexes = []\nfor i, segm in enumerate(output[\"rles\"]):\n    if segm == '':\n        indexes.append(i)\nfor element in sorted(indexes, reverse = True):\n    del output[\"rles\"][element]\n    del output[\"ids\"][element]\n    \nfiles = pd.Series(output[\"ids\"], name='id')\npreds = pd.Series(output[\"rles\"], name='predicted')\nsubmission_df = pd.concat([files, preds], axis=1)\nprint(submission_df.head())\nsubmission_df.to_csv('/kaggle/working/submission.csv', index=False)\n\n\nfor file in test_files:\n    print(\"final masks in file '{:s}': {:d}\".format(file, output[\"ids\"].count(str(file.split('.')[0]))))","metadata":{"execution":{"iopub.status.busy":"2022-01-26T11:29:33.93983Z","iopub.execute_input":"2022-01-26T11:29:33.940328Z","iopub.status.idle":"2022-01-26T11:29:43.365358Z","shell.execute_reply.started":"2022-01-26T11:29:33.940275Z","shell.execute_reply":"2022-01-26T11:29:43.364386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working\n!echo \"\"\n%cd /kaggle/working\n!rm -rf /kaggle/working/data\n!rm -rf /kaggle/working/train\n!rm -rf /kaggle/working/trainannot\n!rm -rf /kaggle/working/val\n!rm -rf /kaggle/working/valannot\n!rm -rf /kaggle/working/carrotssmp\n!rm -rf /kaggle/working/results\n!rm -rf /kaggle/working/testannot\n!echo \"\"\n!ls /kaggle/working","metadata":{},"execution_count":null,"outputs":[]}]}