{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is a notebook I've created for last year's HuBMAP competition. As I'm sure the topic will come up again, I've created this fork to verify that it also works on the current 2022 competition.\nI wrote these routines from scratch, they are an order of magnitude faster than all others I've seen up to now and are also 100% correct.\n\nThese routines are\n* memory-efficient\n* fast\n* definitely correct on a pixel basis for the specific format used in this competition.\n\nSome other routines I've found were off 1 pixel (not that it matters, but aren't we all a bit nitpicky in this line of work ;)) or had other limitations.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport tifffile as tiff","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-23T14:32:02.511492Z","iopub.execute_input":"2022-06-23T14:32:02.511855Z","iopub.status.idle":"2022-06-23T14:32:02.547069Z","shell.execute_reply.started":"2022-06-23T14:32:02.511823Z","shell.execute_reply":"2022-06-23T14:32:02.546202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle2mask(rle, mask_shape):\n    ''' takes a space-delimited RLE string in column-first order\n    and turns it into a 2d boolean numpy array of shape mask_shape '''\n    \n    mask = np.zeros(np.prod(mask_shape), dtype=bool) # 1d mask array\n    rle = np.array(rle.split()).astype(int) # rle values to ints\n    starts = rle[::2]\n    lengths = rle[1::2]\n    for s, l in zip(starts, lengths):\n        mask[s:s+l] = True\n    return mask.reshape(np.flip(mask_shape)).T # flip because of column-first order\n\n\ndef mask2rle(mask):\n    ''' takes a 2d boolean numpy array and turns it into a space-delimited RLE string '''\n    \n    mask = mask.T.reshape(-1) # make 1D, column-first\n    mask = np.pad(mask, 1) # make sure that the 1d mask starts and ends with a 0\n    starts = np.nonzero((~mask[:-1]) & mask[1:])[0] # start points\n    ends = np.nonzero(mask[:-1] & (~mask[1:]))[0] # end points\n    rle = np.empty(2 * starts.size, dtype=int) # interlacing...\n    rle[0::2] = starts # ...starts...\n    rle[1::2] = ends - starts # ...and lengths\n    rle = ' '.join([ str(elem) for elem in rle ]) # turn into space-separated string\n    return rle","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-06-23T14:32:04.953048Z","iopub.execute_input":"2022-06-23T14:32:04.954388Z","iopub.status.idle":"2022-06-23T14:32:04.977146Z","shell.execute_reply.started":"2022-06-23T14:32:04.954326Z","shell.execute_reply":"2022-06-23T14:32:04.975497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's check with the original train RLEs:","metadata":{}},{"cell_type":"code","source":"df_enc = pd.read_csv('../input/hubmap-organ-segmentation/train.csv')\ndf_enc","metadata":{"execution":{"iopub.status.busy":"2022-06-23T14:32:21.444410Z","iopub.execute_input":"2022-06-23T14:32:21.444941Z","iopub.status.idle":"2022-06-23T14:32:21.818200Z","shell.execute_reply.started":"2022-06-23T14:32:21.444902Z","shell.execute_reply":"2022-06-23T14:32:21.817401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We'll take the first image:","metadata":{}},{"cell_type":"code","source":"enc_original = df_enc.loc[0,\"rle\"]\nenc_original[:1000]","metadata":{"execution":{"iopub.status.busy":"2022-06-23T14:33:58.064132Z","iopub.execute_input":"2022-06-23T14:33:58.064839Z","iopub.status.idle":"2022-06-23T14:33:58.072116Z","shell.execute_reply.started":"2022-06-23T14:33:58.064798Z","shell.execute_reply":"2022-06-23T14:33:58.070774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tiff_shape = tiff.TiffFile('../input/hubmap-organ-segmentation/train_images/10044.tiff').pages[0].shape[:2]\ntiff_shape","metadata":{"execution":{"iopub.status.busy":"2022-06-23T14:34:41.838623Z","iopub.execute_input":"2022-06-23T14:34:41.839016Z","iopub.status.idle":"2022-06-23T14:34:41.868853Z","shell.execute_reply.started":"2022-06-23T14:34:41.838968Z","shell.execute_reply":"2022-06-23T14:34:41.867551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fast decoding:","metadata":{}},{"cell_type":"code","source":"%%time\nmask = rle2mask(enc_original, tiff_shape)","metadata":{"execution":{"iopub.status.busy":"2022-06-23T14:34:49.981298Z","iopub.execute_input":"2022-06-23T14:34:49.981704Z","iopub.status.idle":"2022-06-23T14:34:50.011001Z","shell.execute_reply.started":"2022-06-23T14:34:49.981666Z","shell.execute_reply":"2022-06-23T14:34:50.009984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Mask looks ok *and is in the correct numpy orientation*:","metadata":{}},{"cell_type":"code","source":"plt.imshow(mask[::10,::10]);","metadata":{"execution":{"iopub.status.busy":"2022-06-23T14:43:51.921577Z","iopub.execute_input":"2022-06-23T14:43:51.921959Z","iopub.status.idle":"2022-06-23T14:43:52.077592Z","shell.execute_reply.started":"2022-06-23T14:43:51.921917Z","shell.execute_reply":"2022-06-23T14:43:52.076257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fast encoding, no RAM was harmed in the process...","metadata":{}},{"cell_type":"code","source":"%%time\nenc_reencoded = mask2rle(mask)","metadata":{"execution":{"iopub.status.busy":"2022-06-23T14:35:11.946550Z","iopub.execute_input":"2022-06-23T14:35:11.946975Z","iopub.status.idle":"2022-06-23T14:35:11.993269Z","shell.execute_reply.started":"2022-06-23T14:35:11.946935Z","shell.execute_reply":"2022-06-23T14:35:11.992415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And the final check:","metadata":{}},{"cell_type":"code","source":"enc_reencoded[:1000]","metadata":{"execution":{"iopub.status.busy":"2022-06-23T14:35:16.945435Z","iopub.execute_input":"2022-06-23T14:35:16.945983Z","iopub.status.idle":"2022-06-23T14:35:16.952658Z","shell.execute_reply.started":"2022-06-23T14:35:16.945945Z","shell.execute_reply":"2022-06-23T14:35:16.951234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enc_original == enc_reencoded","metadata":{"execution":{"iopub.status.busy":"2022-06-23T14:35:21.505829Z","iopub.execute_input":"2022-06-23T14:35:21.506222Z","iopub.status.idle":"2022-06-23T14:35:21.511731Z","shell.execute_reply.started":"2022-06-23T14:35:21.506185Z","shell.execute_reply":"2022-06-23T14:35:21.511014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"Here we go. Enjoy!","metadata":{}}]}