{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install /kaggle/input/ultralytics/ultralytics-8.0.176-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:39:47.029300Z","iopub.execute_input":"2023-10-21T17:39:47.029638Z","iopub.status.idle":"2023-10-21T17:40:19.757605Z","shell.execute_reply.started":"2023-10-21T17:39:47.029606Z","shell.execute_reply":"2023-10-21T17:40:19.756480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport cv2\nfrom tqdm.notebook import tqdm\nfrom skimage.draw import polygon2mask\nimport json\nimport random\nimport shutil\nfrom PIL import Image\nfrom ultralytics import YOLO","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:40:19.759716Z","iopub.execute_input":"2023-10-21T17:40:19.760006Z","iopub.status.idle":"2023-10-21T17:40:30.371566Z","shell.execute_reply.started":"2023-10-21T17:40:19.759978Z","shell.execute_reply":"2023-10-21T17:40:30.370783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/packages\n!cp -r /kaggle/input/pycocotools/* /kaggle/working/packages\nos.chdir(\"/kaggle/working/packages/pycocotools-2.0.6/\")\n!python setup.py install\n!pip install . --no-index --find-links /kaggle/working/packages/\nos.chdir(\"/kaggle/working\")","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:40:30.372598Z","iopub.execute_input":"2023-10-21T17:40:30.373034Z","iopub.status.idle":"2023-10-21T17:41:15.028128Z","shell.execute_reply.started":"2023-10-21T17:40:30.373008Z","shell.execute_reply":"2023-10-21T17:41:15.027165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from kaggle_secrets import UserSecretsClient\n# user_secrets = UserSecretsClient()\n# api_key = user_secrets.get_secret(\"key\")\n\n# import wandb\n# wandb.login(key=api_key)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:41:15.031486Z","iopub.execute_input":"2023-10-21T17:41:15.031937Z","iopub.status.idle":"2023-10-21T17:41:15.036632Z","shell.execute_reply.started":"2023-10-21T17:41:15.031905Z","shell.execute_reply":"2023-10-21T17:41:15.035577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# determine paths\ntrain_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/train/\"\ntest_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/test/\"\njson_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl\"\nsub_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/sample_submission.csv\"\ntile_meta_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/tile_meta.csv\"\nwsi_meta_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/wsi_meta.csv\"","metadata":{"id":"8_JgFbzcYTmx","execution":{"iopub.status.busy":"2023-10-21T17:41:15.037977Z","iopub.execute_input":"2023-10-21T17:41:15.038304Z","iopub.status.idle":"2023-10-21T17:41:15.057949Z","shell.execute_reply.started":"2023-10-21T17:41:15.038275Z","shell.execute_reply":"2023-10-21T17:41:15.057057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read data\ntile_meta = pd.read_csv(tile_meta_path)\nwsi_meta = pd.read_csv(wsi_meta_path)","metadata":{"id":"TKe-TZZ6YTgS","execution":{"iopub.status.busy":"2023-10-21T17:41:15.059140Z","iopub.execute_input":"2023-10-21T17:41:15.059428Z","iopub.status.idle":"2023-10-21T17:41:15.103392Z","shell.execute_reply.started":"2023-10-21T17:41:15.059406Z","shell.execute_reply":"2023-10-21T17:41:15.102704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample from data\nprint(\"tile_meta\\n\")\nprint(tile_meta.head(2))\nprint(\"wsi_meta\\n\")\nprint(wsi_meta.head(2))","metadata":{"id":"WKAWIywHYTdO","outputId":"dae01334-4d9c-4ad4-bdb5-fc4b54b8f421","execution":{"iopub.status.busy":"2023-10-21T17:41:15.104379Z","iopub.execute_input":"2023-10-21T17:41:15.104632Z","iopub.status.idle":"2023-10-21T17:41:15.120605Z","shell.execute_reply.started":"2023-10-21T17:41:15.104609Z","shell.execute_reply":"2023-10-21T17:41:15.119759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shape\nprint(f\"tile_meta = {tile_meta.shape}\\nwsi_meta = {wsi_meta.shape}\")","metadata":{"id":"VoUyDPgQbsa_","outputId":"2ba9f5be-5f52-42f5-853c-10d6a46fc5b9","execution":{"iopub.status.busy":"2023-10-21T17:41:15.121647Z","iopub.execute_input":"2023-10-21T17:41:15.121952Z","iopub.status.idle":"2023-10-21T17:41:15.126326Z","shell.execute_reply.started":"2023-10-21T17:41:15.121927Z","shell.execute_reply":"2023-10-21T17:41:15.125456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# source_wsi in wsi_meta , tile_meta\nprint(f\"wsi_meta = {wsi_meta['source_wsi'].unique()}\\ntile_meta = {tile_meta['source_wsi'].unique()}\")","metadata":{"id":"b2wg4VD8dMJ8","outputId":"a61a9bab-3844-4ab0-c484-37d97fe47d5b","execution":{"iopub.status.busy":"2023-10-21T17:41:15.127449Z","iopub.execute_input":"2023-10-21T17:41:15.127770Z","iopub.status.idle":"2023-10-21T17:41:15.140528Z","shell.execute_reply.started":"2023-10-21T17:41:15.127740Z","shell.execute_reply":"2023-10-21T17:41:15.139593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mergging data based on source_wsi\ndata = pd.merge(tile_meta , wsi_meta , how  = \"right\", on = \"source_wsi\")\ndata.head()","metadata":{"id":"gOZ2slyuYTah","outputId":"a63c5681-8d4a-4506-b619-ed3563c394a2","execution":{"iopub.status.busy":"2023-10-21T17:41:15.144029Z","iopub.execute_input":"2023-10-21T17:41:15.144283Z","iopub.status.idle":"2023-10-21T17:41:15.175754Z","shell.execute_reply.started":"2023-10-21T17:41:15.144261Z","shell.execute_reply":"2023-10-21T17:41:15.174972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shape\ndata.shape","metadata":{"id":"G6F-z1rYYTXx","outputId":"b475641d-e138-49ba-f875-0293932267ab","execution":{"iopub.status.busy":"2023-10-21T17:41:15.176597Z","iopub.execute_input":"2023-10-21T17:41:15.176855Z","iopub.status.idle":"2023-10-21T17:41:15.182682Z","shell.execute_reply.started":"2023-10-21T17:41:15.176833Z","shell.execute_reply":"2023-10-21T17:41:15.181856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read annotation info\nann = pd.read_json(json_path,lines = True)\nann.head(1)","metadata":{"id":"o-zEYklXcDxN","outputId":"ba90b720-ac2a-4457-db5d-0fcd24ba8936","execution":{"iopub.status.busy":"2023-10-21T17:41:15.183688Z","iopub.execute_input":"2023-10-21T17:41:15.183961Z","iopub.status.idle":"2023-10-21T17:41:19.005845Z","shell.execute_reply.started":"2023-10-21T17:41:15.183939Z","shell.execute_reply":"2023-10-21T17:41:19.004719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shape\nann.shape","metadata":{"id":"C6iSaYiwgD2e","outputId":"cbef48d8-67ec-4866-9a55-321a2491e8a6","execution":{"iopub.status.busy":"2023-10-21T17:41:19.006948Z","iopub.execute_input":"2023-10-21T17:41:19.007232Z","iopub.status.idle":"2023-10-21T17:41:19.012711Z","shell.execute_reply.started":"2023-10-21T17:41:19.007205Z","shell.execute_reply":"2023-10-21T17:41:19.011764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get type\ntype_d = []\nfor i in range(ann.shape[0]):\n  tmp_l = []\n  for j in range(len(ann['annotations'][i])):\n    tmp_l.append(ann['annotations'][i][j]['type'])\n  type_d.append(tmp_l)\nann['type'] = type_d\n\n# get coordinates\ncoordinates_d = []\nfor i in range(ann.shape[0]):\n  tmp_l = []\n  for j in range(len(ann['annotations'][i])):\n    tmp_l +=ann['annotations'][i][j]['coordinates']\n  coordinates_d.append(tmp_l)\nann['coordinates'] = coordinates_d","metadata":{"id":"QzH3zjXvfvcw","execution":{"iopub.status.busy":"2023-10-21T17:41:19.013796Z","iopub.execute_input":"2023-10-21T17:41:19.014089Z","iopub.status.idle":"2023-10-21T17:41:19.399428Z","shell.execute_reply.started":"2023-10-21T17:41:19.014065Z","shell.execute_reply":"2023-10-21T17:41:19.398426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample of data\nann.head(1)","metadata":{"id":"XKryEHkocDuT","outputId":"18f84904-9b81-4543-f21a-c6377256a24d","execution":{"iopub.status.busy":"2023-10-21T17:41:19.400763Z","iopub.execute_input":"2023-10-21T17:41:19.401430Z","iopub.status.idle":"2023-10-21T17:41:19.701696Z","shell.execute_reply.started":"2023-10-21T17:41:19.401397Z","shell.execute_reply":"2023-10-21T17:41:19.700820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merging with annotations\ndata = pd.merge(data,ann,how = 'left', on = 'id')\ndata.head(2)","metadata":{"id":"gp5t36wiiZsu","outputId":"60fda6d2-32ab-4be7-bc79-1ec9f16cebce","execution":{"iopub.status.busy":"2023-10-21T17:41:19.702684Z","iopub.execute_input":"2023-10-21T17:41:19.702946Z","iopub.status.idle":"2023-10-21T17:41:20.132120Z","shell.execute_reply.started":"2023-10-21T17:41:19.702923Z","shell.execute_reply":"2023-10-21T17:41:20.131181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shape\ndata.shape","metadata":{"id":"1y-UEYV1iZqj","outputId":"e986edc6-b82f-4918-ddd7-c302f568dfc0","execution":{"iopub.status.busy":"2023-10-21T17:41:20.133411Z","iopub.execute_input":"2023-10-21T17:41:20.134346Z","iopub.status.idle":"2023-10-21T17:41:20.139494Z","shell.execute_reply.started":"2023-10-21T17:41:20.134318Z","shell.execute_reply":"2023-10-21T17:41:20.138555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read images\ndef read_image(dir = train_path):\n  paths  = []\n  imgs_name = sorted(os.listdir(dir))[:1000]\n  for img_name in imgs_name:\n    paths.append(dir + img_name)\n  images = []\n  for img_p in tqdm(paths):\n    img = cv2.imread(img_p)\n    images.append(img)\n  images = np.array(images)\n  return images\n\ntrain_images = read_image()","metadata":{"id":"GDHxvNPavNGa","outputId":"1c125a07-d399-4f5b-832b-9aa2be05e516","execution":{"iopub.status.busy":"2023-10-21T17:41:20.141063Z","iopub.execute_input":"2023-10-21T17:41:20.141775Z","iopub.status.idle":"2023-10-21T17:41:46.674653Z","shell.execute_reply.started":"2023-10-21T17:41:20.141737Z","shell.execute_reply":"2023-10-21T17:41:46.673841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shape\ntrain_images.shape","metadata":{"id":"0xxv-NwCjCBf","outputId":"04925273-83db-4345-9758-0580ba7f9a51","execution":{"iopub.status.busy":"2023-10-21T17:41:46.675786Z","iopub.execute_input":"2023-10-21T17:41:46.676063Z","iopub.status.idle":"2023-10-21T17:41:46.681840Z","shell.execute_reply.started":"2023-10-21T17:41:46.676039Z","shell.execute_reply":"2023-10-21T17:41:46.680881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create mask\ndef create_mask(dir = train_path):\n  imgs_name = sorted(os.listdir(dir))[:1000]\n  masks = []\n  for n in tqdm(imgs_name):\n    mask  = np.zeros((512,512) , dtype = np.float32)\n    d_ = list(data[data['id'] == n.split('.')[0]]['coordinates'])\n    if len(d_)!=0:\n      for j in d_:\n        for cor in j:\n            coordinates = cor\n            coordinates = [[y,x] for [x,y] in coordinates]\n            polygon = np.array(coordinates)\n            mask += polygon2mask((512,512), polygon)\n            \n    mask = np.where(mask > 1, 1, mask)\n    masks.append(mask)\n  masks = np.array(masks)\n\n  return masks\ntrain_masks = create_mask()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:41:46.682834Z","iopub.execute_input":"2023-10-21T17:41:46.683086Z","iopub.status.idle":"2023-10-21T17:42:14.726348Z","shell.execute_reply.started":"2023-10-21T17:41:46.683053Z","shell.execute_reply":"2023-10-21T17:42:14.725264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_imgs = 20\nfig,axs = plt.subplots(2,n_imgs,figsize = (20,8))\nfig.suptitle('Sample Of Data' , ha = 'center', fontsize = 25 , fontweight = 'bold')\naxs[0,0].set_title('Image')\naxs[1,0].set_title('Mask')\nfor i in range(n_imgs):\n    # image\n    axs[0,i].imshow(train_images[i*10])\n    axs[0,i].axis('off')\n    \n    # mask\n    axs[1,i].imshow(train_masks[i*10])\n    axs[1,i].axis('off')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:42:14.728042Z","iopub.execute_input":"2023-10-21T17:42:14.728354Z","iopub.status.idle":"2023-10-21T17:42:17.338491Z","shell.execute_reply.started":"2023-10-21T17:42:14.728329Z","shell.execute_reply":"2023-10-21T17:42:17.337537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# adjust data for yolov8\nimport base64\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\nroot_dir=\"/kaggle/input/hubmap-hacking-the-human-vasculature\"\n! cd{root_dir}\nannotations=[]\nwith open(os.path.join(root_dir,'polygons.jsonl'), 'r') as f:\n    for line in tqdm(f):\n        annotations.append(json.loads(line))\nprint(len(annotations))\nprint(annotations[0].keys())\n\nannot_dict={} ## creates a list of annotation entries for each id: annptation entrie is a list of dict: dict.keys= ['type','coordinates']\nfor anot in tqdm(annotations):\n    annot_dict[anot['id']]=anot['annotations']\n\nparent_dirpath = \"/kaggle/working/yolov8\"\n\nos.mkdir(parent_dirpath)\nos.mkdir(\"/kaggle/working/temp_images\")\nos.mkdir(\"/kaggle/working/temp_labels\")\n\nos.mkdir(os.path.join(parent_dirpath, \"train\"))\nos.mkdir(os.path.join(parent_dirpath, \"train\", \"images\"))\nos.mkdir(os.path.join(parent_dirpath, \"train\", \"labels\"))\n\nos.mkdir(os.path.join(parent_dirpath, \"test\"))\nos.mkdir(os.path.join(parent_dirpath, \"test\", \"images\"))\nos.mkdir(os.path.join(parent_dirpath, \"test\", \"labels\"))\n\n\ndef tiff_to_jpg(file_name):\n    \n    tiff_image_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/train/\" + str(file_name) + \".tif\"\n    tiff_image = Image.open(tiff_image_path)\n    destination_path = \"/kaggle/working/temp_images/\" + file_name + \".jpg\"\n    tiff_image.save(destination_path, 'JPEG')\n    \n    return 0\n\ndef vertices_to_txt(file_id, annotations, list_of_vertices):\n    \n    file_contents = []\n\n    for i in range(len(annotations)):\n\n        yolo_format = []\n        flag = 1\n\n        if annotations[i]['type'] == 'glomerulus':\n            yolo_format.append(str(1))\n            flag = 1\n        elif annotations[i]['type'] == 'blood_vessel':\n            yolo_format.append(str(0))\n            flag = 1\n        else:\n            flag = 0\n\n\n        if (flag):\n\n            list_of_vertices = annotations[i]['coordinates'][0]\n            for vertex in list_of_vertices:\n                yolo_format.append(str(vertex[0]/512))\n                yolo_format.append(str(vertex[1]/512))\n\n        yolo_format = \" \".join(yolo_format)\n\n        file_contents.append(yolo_format)\n\n    file_name = \"/kaggle/working/temp_labels/\" + str(file_id) + \".txt\"\n\n    with open(file_name, \"w\") as file:\n        if (len(file_contents) == 0):\n            pass\n        elif (len(file_contents) == 1):\n            file.write(str(file_contents[-1]))\n        else:\n            for k in range(len(file_contents)-1):\n                file.write(str(file_contents[k]) + \"\\n\")\n\n            file.write(str(file_contents[-1]))\n            \n    return 0\n\njson_filepath = \"/kaggle/input/hubmap-hacking-the-human-vasculature/polygons.jsonl\"\nfile_ids = []\n\nwith open(json_filepath, 'r') as file:\n    \n    for line in file:\n        data = json.loads(line)\n        file_id = data['id']\n        annotations = data['annotations']\n        list_of_vertices = annotations[0]['coordinates'][0]\n        tiff_to_jpg(file_id)\n        vertices_to_txt(file_id, annotations, list_of_vertices)\n        file_ids.append(file_id)\ntrain_filepath = \"/kaggle/input/hubmap-hacking-the-human-vasculature/train\"\n\nall_images = os.listdir(train_filepath)\nprint(\"No. of images:\", len(all_images))\n\nall_file_ids = []\n\nfor file_name in all_images:\n    file_name = file_name.split('.')\n    all_file_ids.append(file_name[0])\n\nunlabelled_images = list(set(all_file_ids).difference(file_ids))\n\nrandom.shuffle(file_ids)\n\n# Train directory\nfor i in range(0, int(0.8*len(file_ids))):\n    \n    old_path_img = \"/kaggle/working/temp_images/\" + str(file_ids[i]) + \".jpg\"\n    new_path_img = \"/kaggle/working/yolov8/train/images/\" + str(file_ids[i]) + \".jpg\"\n    shutil.copy(old_path_img, new_path_img)\n    \n    old_path_txt = \"/kaggle/working/temp_labels/\" + str(file_ids[i]) + \".txt\"\n    new_path_txt = \"/kaggle/working/yolov8/train/labels/\" + str(file_ids[i]) + \".txt\"\n    shutil.copy(old_path_txt, new_path_txt)\n\n# Test directory\nfor i in range(int(0.8*len(file_ids)), len(file_ids)):\n    \n    old_path_img = \"/kaggle/working/temp_images/\" + str(file_ids[i]) + \".jpg\"\n    new_path_img = \"/kaggle/working/yolov8/test/images/\" + str(file_ids[i]) + \".jpg\"\n    shutil.copy(old_path_img, new_path_img)\n    \n    old_path_txt = \"/kaggle/working/temp_labels/\" + str(file_ids[i]) + \".txt\"\n    new_path_txt = \"/kaggle/working/yolov8/test/labels/\" + str(file_ids[i]) + \".txt\"\n    shutil.copy(old_path_txt, new_path_txt)\n    \nshutil.rmtree(\"/kaggle/working/temp_images\")\nshutil.rmtree(\"/kaggle/working/temp_labels\")\n\n# Creating custom_config.yaml file\nwith open(\"/kaggle/working/custom_config.yaml\", \"w\") as file:\n    file.write(\"path: /kaggle/working/yolov8\" + \"\\n\")\n    file.write(\"train: train/images\" + \"\\n\")\n    file.write(\"val: test/images\" + \"\\n\")\n    file.write(\"test: test/images\" + \"\\n\")\n    file.write(\"nc: 2\" + \"\\n\")\n    file.write(\"names: ['blood_vessel','glomerulus']\")\n## Predict on unlabbeled files and add them to train\n\ndef predict_and_add_images(ssl_epoch,model):\n    \n    images_added = 0\n    \n    for file_id in unlabelled_images:\n\n        tiff_image_path = \"/kaggle/input/hubmap-hacking-the-human-vasculature/train/\" + str(file_id) + \".tif\"\n        tiff_image = Image.open(tiff_image_path)\n        destination_path = \"/kaggle/working/temp_image.jpg\"\n        tiff_image.save(destination_path, 'JPEG')\n\n        results = model.predict(destination_path, verbose=False)\n\n        flag = 1\n        file_contents = []\n\n        for result in results:\n            boxes = result.boxes.conf\n            if len(boxes) != 0:\n                classes = result.boxes.cls\n                masks = result.masks.xyn\n            else:\n                flag = 0\n\n        if (flag):\n            for i in range(len(boxes)):\n                if boxes[i] < 0.2:    ## Set threshold of confidence\n                    flag=0\n                    break\n\n        if(flag):\n            des_img_filepath = os.path.join(\"/kaggle/working/yolov8/train/images/\" + str(file_id) + \".jpg\")\n            shutil.copy(destination_path, des_img_filepath)\n            unlabelled_images.remove(file_id)\n\n            for i in range(len(boxes)):\n\n                yolo_format = []\n\n                if classes[i] == 1:\n                    yolo_format.append(str(1))\n                else:\n                    yolo_format.append(str(0))\n\n                list_of_vertices = masks[i]\n                for vertex in list_of_vertices:\n                    yolo_format.append(str(vertex[0]))\n                    yolo_format.append(str(vertex[1]))\n\n                yolo_format = \" \".join(yolo_format)\n\n                file_contents.append(yolo_format)\n\n            file_name = os.path.join(\"/kaggle/working/yolov8/train/labels/\" + str(file_id) + \".txt\")\n\n            with open(file_name, \"w\") as file:\n                if (len(file_contents) == 1):\n                    file.write(str(file_contents[-1]))\n                else:\n                    for k in range(len(file_contents)-1):\n                        file.write(str(file_contents[k]) + \"\\n\")\n\n                    file.write(str(file_contents[-1]))\n\n            images_added += 1\n\n        flag = 1\n    \n    print(\"#########################################################################\")\n    print(f\" For SSL Epoch {ssl_epoch} Images added to training set: {images_added}\" )\n    print(f\" Number of images left in unlabbeld images {len(unlabelled_images)}\" )\n    print(\"#########################################################################\")\n","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:42:17.339864Z","iopub.execute_input":"2023-10-21T17:42:17.340146Z","iopub.status.idle":"2023-10-21T17:43:01.275211Z","shell.execute_reply.started":"2023-10-21T17:42:17.340121Z","shell.execute_reply":"2023-10-21T17:43:01.274337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model\n# ssl_epochs = 5\n\nmodel = YOLO('/kaggle/input/yolov8-segx-model-1/best.pt')\n\n# for ep in tqdm(range(ssl_epochs)):\n#     print(f\"Starting Train for {ep+1} super_epoch\")\n\n#     results = model.train(data='/kaggle/working/custom_config.yaml',\n#                           epochs = 50,\n#                           imgsz=512,\n# #                           device=[0, 1],\n#                           optimizer='Adam',\n#                           seed=42,\n#                           close_mosaic=0,\n#                           mask_ratio=1,\n#                           val=False,\n#                           verbose=False,\n#                           # for augmentation\n#                           degrees=90,\n#                           translate=0.1,\n#                           scale=0.5,\n#                           flipud=0.5,\n#                           fliplr=0.5)\n    \n#     predict_and_add_images(ep,model)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-21T17:43:01.276553Z","iopub.execute_input":"2023-10-21T17:43:01.277314Z","iopub.status.idle":"2023-10-21T17:43:02.931397Z","shell.execute_reply.started":"2023-10-21T17:43:01.277288Z","shell.execute_reply":"2023-10-21T17:43:02.930368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_binary_mask(mask: np.ndarray) -> t.Text:\n    \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n\n    # check input mask --\n    if mask.dtype != np.bool_:\n        raise ValueError(\n            \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n            mask.dtype)\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(\n            \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n            mask.shape)\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:43:02.932900Z","iopub.execute_input":"2023-10-21T17:43:02.933274Z","iopub.status.idle":"2023-10-21T17:43:02.941922Z","shell.execute_reply.started":"2023-10-21T17:43:02.933216Z","shell.execute_reply":"2023-10-21T17:43:02.941063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONF = 0.001\nSIZE = 512\nresults = model.predict(\"/kaggle/input/hubmap-hacking-the-human-vasculature/test/\", \n                           save=True, imgsz=SIZE, conf=CONF,iou=0.6,save_conf=True,device=0,stream=True)\n\nids= []\nprediction_strings = []\nh = []\nw = []\nfor j,result in enumerate(results):\n    img_name = result.path.split(\"/\")[-1].replace(\".tif\",\"\")\n    ids.append(img_name)\n    \n    h.append(result.orig_shape[0])\n    w.append(result.orig_shape[1])\n    \n    \n    #if img_name in dict_of_tiles.keys():\n    #    annotations = dict_of_tiles[img_name]\n    #    glomerulus_mask = get_glomerulus_mask(annotations)\n    #else:\n    #    annotations = []\n    #    glomerulus_mask = np.ones(shape=(512,512)).astype(bool)\n\n\n    \n    try:\n        conf = result.boxes.conf.cpu().numpy()\n        masks = result.masks.data.cpu()\n        if SIZE!=512:\n            masks = torch.nn.functional.interpolate(\n                masks[None,:,:,:],\n                size=512,\n                mode=\"bicubic\",\n                align_corners=False\n            )[0]\n\n        \n        \n        \n        masks = masks.numpy()\n        pred_=[]\n\n        for i in range(len(masks)):\n            c = conf[i]\n            #& \n            rle = encode_binary_mask(np.where((masks[i])>=CONF,True,False)).decode('utf-8')\n            ##  filter out glomerulus_mask\n            #rle = encode_binary_mask(np.where((masks[i])>=CONF,True,False)&glomerulus_mask).decode('utf-8')\n\n            pred_.append(f\"0 {c} {rle}\")\n\n        #f_pred = \n        prediction_strings.append(\" \".join(pred_))\n    except Exception as e:\n        prediction_strings.append(\"\")\n        print(e)\n","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:43:02.943254Z","iopub.execute_input":"2023-10-21T17:43:02.943590Z","iopub.status.idle":"2023-10-21T17:43:11.773719Z","shell.execute_reply.started":"2023-10-21T17:43:02.943559Z","shell.execute_reply":"2023-10-21T17:43:11.772831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame()\nsub[\"id\"]=ids\nsub[\"height\"]=h\nsub[\"width\"]=w\nsub[\"prediction_string\"]=prediction_strings\n\nsub.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:43:11.774841Z","iopub.execute_input":"2023-10-21T17:43:11.775101Z","iopub.status.idle":"2023-10-21T17:43:11.788873Z","shell.execute_reply.started":"2023-10-21T17:43:11.775077Z","shell.execute_reply":"2023-10-21T17:43:11.788159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T17:43:11.789863Z","iopub.execute_input":"2023-10-21T17:43:11.790142Z","iopub.status.idle":"2023-10-21T17:43:11.801889Z","shell.execute_reply.started":"2023-10-21T17:43:11.790117Z","shell.execute_reply":"2023-10-21T17:43:11.800951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}