{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Although mAP metric is popular, I think we still need to write a new notebook to compute CV score\n1. Most of the mAP calculators are integrated within particular frameworks such as detectron2, yolo, mmdet, etc... It might lack flexibility if we want to ensemble various models.\n2. Some standalone codes only compute box-mAP, not segm-mAP\n\n=> So I reimplement the code to compute mAP for this competition. **Note that it only support single class at the moment**. I use my Detectron2 model to make predictions as an example and I also compare with Detectron2 built-in CocoEvaluator","metadata":{}},{"cell_type":"code","source":"!pip install  -q /kaggle/input/detectron2-wheel/detectron2/detectron2-0.6-cp310-cp310-linux_x86_64.whl --no-index --find-links=/kaggle/input/detectron2-wheel/detectron2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-25T08:08:43.592809Z","iopub.execute_input":"2023-06-25T08:08:43.593161Z","iopub.status.idle":"2023-06-25T08:08:55.232953Z","shell.execute_reply.started":"2023-06-25T08:08:43.593124Z","shell.execute_reply":"2023-06-25T08:08:55.231704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import detectron2\nfrom pathlib import Path\nimport random, cv2, os\nimport matplotlib.pyplot as plt\n# import some common detectron2 utilities\nfrom detectron2 import model_zoo\nfrom detectron2.engine import DefaultPredictor, DefaultTrainer\nfrom detectron2.config import get_cfg, CfgNode\nfrom detectron2.utils.visualizer import Visualizer, ColorMode\nfrom detectron2.data import MetadataCatalog, DatasetCatalog\nfrom detectron2.data import build_detection_train_loader, build_detection_test_loader, DatasetMapper\nfrom detectron2.data.datasets import register_coco_instances","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:08:55.234984Z","iopub.execute_input":"2023-06-25T08:08:55.235912Z","iopub.status.idle":"2023-06-25T08:08:57.220356Z","shell.execute_reply.started":"2023-06-25T08:08:55.235864Z","shell.execute_reply":"2023-06-25T08:08:57.219362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom tqdm.auto import tqdm\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport base64\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\nimport json","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:08:57.221778Z","iopub.execute_input":"2023-06-25T08:08:57.222504Z","iopub.status.idle":"2023-06-25T08:08:57.293361Z","shell.execute_reply.started":"2023-06-25T08:08:57.222469Z","shell.execute_reply":"2023-06-25T08:08:57.292333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load model","metadata":{}},{"cell_type":"code","source":"IMG_DIR = \"/kaggle/input/hubmap-hacking-the-human-vasculature/train\"\nMODEL_PATH = \"/kaggle/input/hubmap-v0-v0-val-ds2-wsi1-2/model_0005499.pth\"\nBLOOD_VESSEL_CLS = 0","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:08:57.296736Z","iopub.execute_input":"2023-06-25T08:08:57.297147Z","iopub.status.idle":"2023-06-25T08:08:57.302338Z","shell.execute_reply.started":"2023-06-25T08:08:57.297109Z","shell.execute_reply":"2023-06-25T08:08:57.301221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = get_cfg()\n\ncfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 2\n\ncfg.INPUT.MAX_SIZE_TEST = 1000\ncfg.INPUT.MAX_SIZE_TRAIN = 1000\ncfg.INPUT.MIN_SIZE_TEST = 512\ncfg.INPUT.MIN_SIZE_TRAIN = (512, )\n\ncfg.MODEL.WEIGHTS = MODEL_PATH\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.001  # set threshold for this model\ncfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = 0.5\ncfg.MODEL.DEVICE = 'cuda:0'\n\npredictor = DefaultPredictor(cfg)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:08:57.304942Z","iopub.execute_input":"2023-06-25T08:08:57.306191Z","iopub.status.idle":"2023-06-25T08:08:59.952275Z","shell.execute_reply.started":"2023-06-25T08:08:57.306156Z","shell.execute_reply":"2023-06-25T08:08:59.951171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\npath = '/kaggle/input/hubmap-hacking-the-human-vasculature/test/72e40acccadf.tif'\nimg = cv2.imread(path)\noutputs = predictor(img)\n\nplt.figure(figsize = (10,10))\n\n\nv = Visualizer(img[:,:,::-1], None, scale=1.2)\nout = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\nplt.imshow(out.get_image()[:, :, ::-1])","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:08:59.953931Z","iopub.execute_input":"2023-06-25T08:08:59.954324Z","iopub.status.idle":"2023-06-25T08:09:04.065076Z","shell.execute_reply.started":"2023-06-25T08:08:59.954288Z","shell.execute_reply":"2023-06-25T08:09:04.064145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## segm-mAP code","metadata":{}},{"cell_type":"code","source":"import pycocotools.mask as mask_util","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:09:04.066055Z","iopub.execute_input":"2023-06-25T08:09:04.066956Z","iopub.status.idle":"2023-06-25T08:09:04.072020Z","shell.execute_reply.started":"2023-06-25T08:09:04.066922Z","shell.execute_reply":"2023-06-25T08:09:04.071042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(predictor, img):\n    pred_instances = predictor(img)\n    pred_classes = pred_instances['instances'].pred_classes.cpu().numpy()\n    keep = pred_classes == BLOOD_VESSEL_CLS\n    \n    pred_masks = pred_instances['instances'].pred_masks.cpu().numpy()\n    pred_masks = pred_masks[keep]\n    \n    scores = pred_instances['instances'].scores.cpu().numpy()\n    scores = scores[keep]\n    \n    return pred_masks, pred_classes, scores","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:09:04.073495Z","iopub.execute_input":"2023-06-25T08:09:04.074156Z","iopub.status.idle":"2023-06-25T08:09:04.083271Z","shell.execute_reply.started":"2023-06-25T08:09:04.074127Z","shell.execute_reply":"2023-06-25T08:09:04.082487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def voc_ap(rec, prec):\n    \"\"\"\n    --- Official matlab code VOC2012---\n    mrec=[0 ; rec ; 1];\n    mpre=[0 ; prec ; 0];\n    for i=numel(mpre)-1:-1:1\n            mpre(i)=max(mpre(i),mpre(i+1));\n    end\n    i=find(mrec(2:end)~=mrec(1:end-1))+1;\n    ap=sum((mrec(i)-mrec(i-1)).*mpre(i));\n    \"\"\"\n    rec.insert(0, 0.0) # insert 0.0 at begining of list\n    rec.append(1.0) # insert 1.0 at end of list\n    mrec = rec[:]\n    prec.insert(0, 0.0) # insert 0.0 at begining of list\n    prec.append(0.0) # insert 0.0 at end of list\n    mpre = prec[:]\n    \"\"\"\n     This part makes the precision monotonically decreasing\n        (goes from the end to the beginning)\n        matlab: for i=numel(mpre)-1:-1:1\n                    mpre(i)=max(mpre(i),mpre(i+1));\n    \"\"\"\n    # matlab indexes start in 1 but python in 0, so I have to do:\n    #     range(start=(len(mpre) - 2), end=0, step=-1)\n    # also the python function range excludes the end, resulting in:\n    #     range(start=(len(mpre) - 2), end=-1, step=-1)\n    for i in range(len(mpre)-2, -1, -1):\n        mpre[i] = max(mpre[i], mpre[i+1])\n    \"\"\"\n     This part creates a list of indexes where the recall changes\n        matlab: i=find(mrec(2:end)~=mrec(1:end-1))+1;\n    \"\"\"\n    i_list = []\n    for i in range(1, len(mrec)):\n        if mrec[i] != mrec[i-1]:\n            i_list.append(i) # if it was matlab would be i + 1\n    \"\"\"\n     The Average Precision (AP) is the area under the curve\n        (numerical integration)\n        matlab: ap=sum((mrec(i)-mrec(i-1)).*mpre(i));\n    \"\"\"\n    ap = 0.0\n    for i in i_list:\n        ap += ((mrec[i]-mrec[i-1])*mpre[i])\n    return ap, mrec, mpre\n","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:09:04.084612Z","iopub.execute_input":"2023-06-25T08:09:04.085220Z","iopub.status.idle":"2023-06-25T08:09:04.098217Z","shell.execute_reply.started":"2023-06-25T08:09:04.085187Z","shell.execute_reply":"2023-06-25T08:09:04.097309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MAPCalculatorSingleClass:\n    def __init__(self, thresholds=[0.6]):\n        self.ious = []\n        self.confidences = []\n        self.image_ids = []\n        self.current_img_id = 0\n        \n        self.GT = 0\n        \n    def accumulate(self, ious, confidences, num_gt):\n        assert len(ious) == len(confidences)\n        self.ious.extend([x for x in ious])\n        self.confidences.extend(confidences)\n        self.image_ids.extend([self.current_img_id]*len(confidences))\n        self.current_img_id += 1\n        self.GT += num_gt\n        \n        \n    def evaluate(self, thresholds=[0.6], vis=False):        \n        # sort by confidence descending\n        sorted_inds = np.argsort(self.confidences)[::-1]\n        \n        results = dict()\n        \n        for th in thresholds:\n            accum_tp = 0\n            accum_fp = 0\n            \n            TP = []\n            FP = []\n            list_gts = dict()\n            \n            NPREDS = []\n            count = 0\n            for ind in sorted_inds:\n                iou_row = self.ious[ind]\n                img_id = self.image_ids[ind]\n                \n                matched_inds = np.where(iou_row >= th)[0]\n                best_gt_ind = -1\n                best_iou = 0\n                for gt_ind in matched_inds:\n                    iou = iou_row[gt_ind]\n                    if iou > best_iou and list_gts.get((img_id, gt_ind)) is None:\n                        best_iou = iou\n                        best_gt_ind = gt_ind\n                \n                if best_gt_ind != -1:\n                    list_gts[(img_id, best_gt_ind)] = True\n                    accum_tp += 1\n                else:\n                    accum_fp += 1\n                    \n                count += 1\n                NPREDS.append(count)\n                    \n                TP.append(accum_tp)\n                FP.append(accum_fp)\n                \n            PR = []\n            REC = []\n            \n            for tp, fp in zip(TP, FP):\n                pr = tp / (tp+fp)\n                rec = tp / self.GT\n                PR.append(pr)\n                REC.append(rec)\n                \n            if vis:\n                plt.figure()\n                plt.plot(REC, PR, '-o', label='precision-recall curve')\n                \n            ap, mrec, mpre = voc_ap(REC, PR)\n            \n            if vis:\n                plt.plot(mrec, mpre, '--', label='interpolation')\n                plt.legend()\n                plt.title('Precision recall curve at threshold:'+str(np.round(th, 2)))\n                plt.show()\n            \n            results[th] = ap\n    \n        return np.mean(list(results.values())), results","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:09:04.099709Z","iopub.execute_input":"2023-06-25T08:09:04.100363Z","iopub.status.idle":"2023-06-25T08:09:04.119388Z","shell.execute_reply.started":"2023-06-25T08:09:04.100331Z","shell.execute_reply":"2023-06-25T08:09:04.118352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Compare with Detectron2 mAP@0.5:0.95 CocoEvaluator","metadata":{}},{"cell_type":"code","source":"JSON_ANN = '/kaggle/input/hubmap-json/ds2_wsi2.json'\nwith open(JSON_ANN, 'r') as f:\n    annotations = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:09:04.120834Z","iopub.execute_input":"2023-06-25T08:09:04.121191Z","iopub.status.idle":"2023-06-25T08:09:04.294329Z","shell.execute_reply.started":"2023-06-25T08:09:04.121156Z","shell.execute_reply":"2023-06-25T08:09:04.293298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotations['categories'] # in json annotations, class id starts from 1","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:14:11.396208Z","iopub.execute_input":"2023-06-25T08:14:11.396600Z","iopub.status.idle":"2023-06-25T08:14:11.402791Z","shell.execute_reply.started":"2023-06-25T08:14:11.396568Z","shell.execute_reply":"2023-06-25T08:14:11.401809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"JSON_BLOOD_VESSEL_CLS = 1","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:15:17.624170Z","iopub.execute_input":"2023-06-25T08:15:17.624569Z","iopub.status.idle":"2023-06-25T08:15:17.629549Z","shell.execute_reply.started":"2023-06-25T08:15:17.624541Z","shell.execute_reply":"2023-06-25T08:15:17.628392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Our custom code","metadata":{}},{"cell_type":"code","source":"mAP_calc = MAPCalculatorSingleClass()\n\nfor img_meta in tqdm(annotations['images']):\n    width, height = img_meta['width'], img_meta['height']\n    img_path = os.path.join(IMG_DIR, img_meta['file_name'])\n    img = cv2.imread(img_path)\n    \n    # when calculating IOU using pycocotools.mask, we need to encode both\n    # groundtruth and prediction\n    \n    # make ground truth    \n    targs = []\n    enc_targs = []\n    for ann in annotations['annotations']:\n        if ann['image_id'] == img_meta['id'] and ann['category_id'] == JSON_BLOOD_VESSEL_CLS:\n            seg = ann['segmentation']\n            if type(seg) == list: # if polygon, need to convert to RLE\n                seg = coco_mask.frPyObjects(seg, height, width)[0]\n            targs.append(seg)\n        enc_targs = targs\n    num_gts = len(enc_targs)\n        \n    # make prediction\n    pred_masks, pred_classes, scores = predict(predictor, img)\n    # pred_masks: list of numpy array shape H*W \n    enc_preds = [mask_util.encode(np.asarray(p, order='F')) for p in pred_masks]\n    \n    # calculate iou\n    if len(enc_targs) > 0:\n        ious = mask_util.iou(enc_preds, enc_targs, [0]*len(enc_targs))\n    else:\n        ious = np.array([[0]]*len(enc_preds))\n    \n    # acummulate predictions\n    mAP_calc.accumulate(ious, scores, num_gts)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:15:26.482547Z","iopub.execute_input":"2023-06-25T08:15:26.482966Z","iopub.status.idle":"2023-06-25T08:15:52.621466Z","shell.execute_reply.started":"2023-06-25T08:15:26.482934Z","shell.execute_reply":"2023-06-25T08:15:52.620465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mAP, detail_scores = mAP_calc.evaluate(thresholds=np.arange(0.5, 1.0, 0.05))\nprint('segm-mAP@0.5:0.95 by custom code:', mAP)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:15:52.623532Z","iopub.execute_input":"2023-06-25T08:15:52.624148Z","iopub.status.idle":"2023-06-25T08:15:53.722294Z","shell.execute_reply.started":"2023-06-25T08:15:52.624113Z","shell.execute_reply":"2023-06-25T08:15:53.720548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_name = 'tmp'\nregister_coco_instances('tmp', {}, JSON_ANN, IMG_DIR)\nmetadata = MetadataCatalog.get(ds_name)\n\nds = DatasetCatalog.get(ds_name)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:15:53.723850Z","iopub.execute_input":"2023-06-25T08:15:53.724230Z","iopub.status.idle":"2023-06-25T08:15:53.840335Z","shell.execute_reply.started":"2023-06-25T08:15:53.724198Z","shell.execute_reply":"2023-06-25T08:15:53.839033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.evaluation import COCOEvaluator, inference_on_dataset\nfrom detectron2.data import build_detection_test_loader\n\nevaluator = COCOEvaluator(ds_name,  (\"segm\", ), False, output_dir=\"./tmp\")\nval_loader = build_detection_test_loader(cfg, ds_name)\n\nprint('segm-mAP@0.5:0.95 by detectron2 CocoEvaluator')\nprint(inference_on_dataset(predictor.model, val_loader, evaluator))","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:09:32.343154Z","iopub.execute_input":"2023-06-25T08:09:32.343593Z","iopub.status.idle":"2023-06-25T08:10:05.876357Z","shell.execute_reply.started":"2023-06-25T08:09:32.343559Z","shell.execute_reply":"2023-06-25T08:10:05.875307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Comparision\n- Our custom code outputs AP=0.1777 and Detectron2 CocoEvaluator outputs AP-blood_vessel = 0.1791","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Competition metric\n- In this competition, the metric is segm-mAP@0.6","metadata":{}},{"cell_type":"code","source":"# We test on the above dataset and get the CV score and visualize precision-recall curve\nmAP, detail_scores = mAP_calc.evaluate(thresholds=[0.6], vis=True)\nprint('CV (mAP@0.6):', mAP)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:10:05.878063Z","iopub.execute_input":"2023-06-25T08:10:05.878674Z","iopub.status.idle":"2023-06-25T08:10:06.369926Z","shell.execute_reply.started":"2023-06-25T08:10:05.878633Z","shell.execute_reply":"2023-06-25T08:10:06.369021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make submission ","metadata":{}},{"cell_type":"code","source":"def encode_binary_mask(mask: np.ndarray) -> t.Text:\n    # check input mask --\n    if mask.dtype != bool:\n        raise ValueError(\n            \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n            mask.dtype)\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(\n            \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n            mask.shape)\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:10:06.371362Z","iopub.execute_input":"2023-06-25T08:10:06.372054Z","iopub.status.idle":"2023-06-25T08:10:06.380501Z","shell.execute_reply.started":"2023-06-25T08:10:06.372020Z","shell.execute_reply":"2023-06-25T08:10:06.379480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/test\"\nsample_submission = pd.read_csv('/kaggle/input/hubmap-hacking-the-human-vasculature/sample_submission.csv')\nids = []\nheights = []\nwidths = []\nprediction_strings = []\n\n\nfor test_name in tqdm(os.listdir(test_path)):\n    inp_path = os.path.join(test_path, test_name)\n    img = cv2.imread(inp_path)\n    h, w, _ = img.shape\n    \n    pred_masks, pred_classes, scores = predict(predictor, img)\n    \n   \n    pred_string = \"\"\n    for i, mask in enumerate(pred_masks):\n        encoded = encode_binary_mask(mask)\n        \n        if i == 0:\n            pred_string += f\"{int(pred_classes[i])} {scores[i]} {encoded.decode('utf-8')}\"\n        else:\n            pred_string += f\" {int(pred_classes[i])} {scores[i]} {encoded.decode('utf-8')}\"\n            \n    ids.append(test_name.split('.')[0])\n    heights.append(h)\n    widths.append(w)\n    prediction_strings.append(pred_string)\n    \n#     break","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:10:06.381943Z","iopub.execute_input":"2023-06-25T08:10:06.382355Z","iopub.status.idle":"2023-06-25T08:10:06.521746Z","shell.execute_reply.started":"2023-06-25T08:10:06.382324Z","shell.execute_reply":"2023-06-25T08:10:06.520752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame({'id':ids, 'height':heights, 'width':widths, \n                    'prediction_string':prediction_strings})","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:10:06.523239Z","iopub.execute_input":"2023-06-25T08:10:06.524174Z","iopub.status.idle":"2023-06-25T08:10:06.530216Z","shell.execute_reply.started":"2023-06-25T08:10:06.524137Z","shell.execute_reply":"2023-06-25T08:10:06.529290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:10:06.531568Z","iopub.execute_input":"2023-06-25T08:10:06.532313Z","iopub.status.idle":"2023-06-25T08:10:06.547608Z","shell.execute_reply.started":"2023-06-25T08:10:06.532281Z","shell.execute_reply":"2023-06-25T08:10:06.546586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-25T08:10:06.549177Z","iopub.execute_input":"2023-06-25T08:10:06.549538Z","iopub.status.idle":"2023-06-25T08:10:06.556666Z","shell.execute_reply.started":"2023-06-25T08:10:06.549507Z","shell.execute_reply":"2023-06-25T08:10:06.555880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}