{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%config Completer.use_jedi = False","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:03:14.103889Z","iopub.execute_input":"2021-12-27T23:03:14.104798Z","iopub.status.idle":"2021-12-27T23:03:14.141974Z","shell.execute_reply.started":"2021-12-27T23:03:14.104667Z","shell.execute_reply":"2021-12-27T23:03:14.141232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -Uqqq pycocotools","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:03:14.472097Z","iopub.execute_input":"2021-12-27T23:03:14.472401Z","iopub.status.idle":"2021-12-27T23:03:36.464678Z","shell.execute_reply.started":"2021-12-27T23:03:14.472367Z","shell.execute_reply":"2021-12-27T23:03:36.463489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:03:36.466564Z","iopub.execute_input":"2021-12-27T23:03:36.466838Z","iopub.status.idle":"2021-12-27T23:03:36.471987Z","shell.execute_reply.started":"2021-12-27T23:03:36.466805Z","shell.execute_reply":"2021-12-27T23:03:36.470895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/all-shsy5y-images/shs_train_df.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:03:36.473275Z","iopub.execute_input":"2021-12-27T23:03:36.473553Z","iopub.status.idle":"2021-12-27T23:03:38.100193Z","shell.execute_reply.started":"2021-12-27T23:03:36.473512Z","shell.execute_reply":"2021-12-27T23:03:38.099227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Based on: https://www.kaggle.com/eigrad/convert-rle-to-bounding-box-x0-y0-x1-y1\ndef rle2mask(rle, img_w, img_h):\n    \n    ## transforming the string into an array of shape (2, N)\n    array = np.fromiter(rle.split(), dtype = np.uint)\n    array = array.reshape((-1,2)).T\n    array[0] = array[0] - 1\n    \n    ## decompressing the rle encoding (ie, turning [3, 1, 10, 2] into [3, 4, 10, 11, 12])\n    # for faster mask construction\n    starts, lenghts = array\n    mask_decompressed = np.concatenate([np.arange(s, s + l, dtype = np.uint) for s, l in zip(starts, lenghts)])\n\n    ## Building the binary mask\n    msk_img = np.zeros(img_w * img_h, dtype = np.uint8)\n    msk_img[mask_decompressed] = 1\n    msk_img = msk_img.reshape((img_h, img_w))\n    msk_img = np.asfortranarray(msk_img) ## This is important so pycocotools can handle this object\n    \n    return msk_img","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:03:53.127506Z","iopub.execute_input":"2021-12-27T23:03:53.127836Z","iopub.status.idle":"2021-12-27T23:03:53.136711Z","shell.execute_reply.started":"2021-12-27T23:03:53.127802Z","shell.execute_reply":"2021-12-27T23:03:53.135842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm\nfrom pycocotools import mask as maskUtils\nfrom joblib import Parallel, delayed\n\ndef annotate(idx, row, cat_ids):\n        mask = rle2mask(row['annotation'], row['width'], row['height']) # Binary mask\n        c_rle = maskUtils.encode(mask) # Encoding it back to rle (coco format)\n        c_rle['counts'] = c_rle['counts'].decode('utf-8') # converting from binary to utf-8\n        area = maskUtils.area(c_rle).item() # calculating the area\n        bbox = maskUtils.toBbox(c_rle).astype(int).tolist() # calculating the bboxes\n        annotation = {\n            'segmentation': c_rle,\n            'bbox': bbox,\n            'area': area,\n            'image_id':row['id'], \n            'category_id':cat_ids[row['cell_type']], \n            'iscrowd':0, \n            'id':idx\n        }\n        return annotation\n    \ndef coco_structure(df, workers = 4):\n    \n    ## Building the header\n    cat_ids = {name:id+1 for id, name in enumerate(df.cell_type.unique())}    \n    cats =[{'name':name, 'id':id} for name,id in cat_ids.items()]\n    images = [{'id':id, 'width':row.width, 'height':row.height, 'file_name':f'../input/shsy5y-images/new_shs_images/{id}.png'} for id,row in df.groupby('id').agg('first').iterrows()]\n    \n    ## Building the annotations\n    annotations = Parallel(n_jobs=workers)(delayed(annotate)(idx, row, cat_ids) for idx, row in tqdm(df.iterrows(), total = len(df)))\n        \n    return {'categories':cats, 'images':images, 'annotations':annotations}","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:05:23.062763Z","iopub.execute_input":"2021-12-27T23:05:23.063074Z","iopub.status.idle":"2021-12-27T23:05:23.075669Z","shell.execute_reply.started":"2021-12-27T23:05:23.063038Z","shell.execute_reply":"2021-12-27T23:05:23.074674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.id.unique()","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:05:23.348789Z","iopub.execute_input":"2021-12-27T23:05:23.349480Z","iopub.status.idle":"2021-12-27T23:05:23.353825Z","shell.execute_reply.started":"2021-12-27T23:05:23.349425Z","shell.execute_reply":"2021-12-27T23:05:23.353105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ndf_train,df_val = train_test_split(df.id.unique(),test_size=.03,random_state=42,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:05:23.842733Z","iopub.execute_input":"2021-12-27T23:05:23.843352Z","iopub.status.idle":"2021-12-27T23:05:24.888300Z","shell.execute_reply.started":"2021-12-27T23:05:23.843289Z","shell.execute_reply":"2021-12-27T23:05:24.887379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df[df.id.isin(df_train)]\ndf_val = df[df.id.isin(df_val)]","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:05:25.517533Z","iopub.execute_input":"2021-12-27T23:05:25.517832Z","iopub.status.idle":"2021-12-27T23:05:25.546396Z","shell.execute_reply.started":"2021-12-27T23:05:25.517803Z","shell.execute_reply":"2021-12-27T23:05:25.545465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json,itertools\nroot_train = coco_structure(df_train)\nroot_val = coco_structure(df_val)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:05:27.505849Z","iopub.execute_input":"2021-12-27T23:05:27.506134Z","iopub.status.idle":"2021-12-27T23:08:01.230137Z","shell.execute_reply.started":"2021-12-27T23:05:27.506102Z","shell.execute_reply":"2021-12-27T23:08:01.229371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nwith open('annotations_train.json', 'w', encoding='utf-8') as f:\n    json.dump(root_train, f, ensure_ascii=True, indent=4)\n\n\n\nwith open('annotations_val.json', 'w', encoding='utf-8') as f:\n    json.dump(root_val, f, ensure_ascii=True, indent=4)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-12-27T23:08:05.794063Z","iopub.execute_input":"2021-12-27T23:08:05.794416Z","iopub.status.idle":"2021-12-27T23:08:12.940103Z","shell.execute_reply.started":"2021-12-27T23:08:05.794380Z","shell.execute_reply":"2021-12-27T23:08:12.939200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# from pycocotools.coco import COCO\n# import matplotlib.pyplot as plt\n# from pathlib import Path\n# from PIL import Image\n\n","metadata":{"execution":{"iopub.status.busy":"2021-12-26T02:31:32.463921Z","iopub.execute_input":"2021-12-26T02:31:32.464173Z","iopub.status.idle":"2021-12-26T02:31:32.470625Z","shell.execute_reply.started":"2021-12-26T02:31:32.464143Z","shell.execute_reply":"2021-12-26T02:31:32.469714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataDir=Path('../input/image-filters/data')\n# annFile = Path('./annotations_train.json')\n# coco = COCO(annFile)\n# imgIds = coco.getImgIds()","metadata":{"execution":{"iopub.status.busy":"2021-12-26T02:31:32.820484Z","iopub.execute_input":"2021-12-26T02:31:32.820773Z","iopub.status.idle":"2021-12-26T02:31:35.743563Z","shell.execute_reply.started":"2021-12-26T02:31:32.820742Z","shell.execute_reply":"2021-12-26T02:31:35.742494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-12-26T02:35:02.018115Z","iopub.execute_input":"2021-12-26T02:35:02.018463Z","iopub.status.idle":"2021-12-26T02:35:02.025758Z","shell.execute_reply.started":"2021-12-26T02:35:02.018423Z","shell.execute_reply":"2021-12-26T02:35:02.024873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}