{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"最後のSettingだけ設定して、、RunAllするだけでデータセットができます。","metadata":{"papermill":{"duration":0.003105,"end_time":"2022-08-26T00:59:26.967229","exception":false,"start_time":"2022-08-26T00:59:26.964124","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# import / config","metadata":{"papermill":{"duration":0.002106,"end_time":"2022-08-26T00:59:26.971715","exception":false,"start_time":"2022-08-26T00:59:26.969609","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import gc\nimport os\nimport cv2\nimport torch\nimport zipfile\nimport rasterio\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport tifffile as tiff\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nfrom rasterio.windows import Window\nfrom torch.utils.data import Dataset","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":2.339796,"end_time":"2022-08-26T00:59:29.313774","exception":false,"start_time":"2022-08-26T00:59:26.973978","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-26T08:35:31.768473Z","iopub.execute_input":"2022-08-26T08:35:31.769002Z","iopub.status.idle":"2022-08-26T08:35:34.836107Z","shell.execute_reply.started":"2022-08-26T08:35:31.768905Z","shell.execute_reply":"2022-08-26T08:35:34.834981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUT_TRAIN = 'train.zip'\nOUT_MASKS = 'masks.zip'\nMASKS = '../input/hubmap-organ-segmentation/train.csv'\nDATA = '../input/hubmap-organ-segmentation/train_images'","metadata":{"papermill":{"duration":0.010223,"end_time":"2022-08-26T00:59:29.326700","exception":false,"start_time":"2022-08-26T00:59:29.316477","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-26T08:35:34.838379Z","iopub.execute_input":"2022-08-26T08:35:34.839096Z","iopub.status.idle":"2022-08-26T08:35:34.845535Z","shell.execute_reply.started":"2022-08-26T08:35:34.839050Z","shell.execute_reply":"2022-08-26T08:35:34.844447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ","metadata":{"papermill":{"duration":0.002428,"end_time":"2022-08-26T00:59:29.331877","exception":false,"start_time":"2022-08-26T00:59:29.329449","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Create Dataset","metadata":{"papermill":{"duration":0.002125,"end_time":"2022-08-26T00:59:29.336446","exception":false,"start_time":"2022-08-26T00:59:29.334321","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# train data (mask data) を 読み込む\ndf_masks = pd.read_csv(MASKS)[['id','organ','rle']].set_index('id')\n\nprint(df_masks.shape)\ndf_masks.head()","metadata":{"papermill":{"duration":0.333617,"end_time":"2022-08-26T00:59:29.672363","exception":false,"start_time":"2022-08-26T00:59:29.338746","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-26T08:35:34.847436Z","iopub.execute_input":"2022-08-26T08:35:34.848070Z","iopub.status.idle":"2022-08-26T08:35:35.247260Z","shell.execute_reply.started":"2022-08-26T08:35:34.848035Z","shell.execute_reply":"2022-08-26T08:35:35.245966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# functions to convert encoding to mask and mask to encoding\ndef enc2mask(encs, shape):\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for m,enc in enumerate(encs):\n        if isinstance(enc, np.float_) and np.isnan(enc): continue\n        s = enc.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1 + m\n    return img.reshape(shape).T\n\ndef mask2enc(mask, n=1):\n    pixels = mask.T.flatten()\n    encs = []\n    for i in range(1,n+1):\n        p = (pixels == i).astype(np.int8)\n        if p.sum() == 0: encs.append(np.nan)\n        else:\n            p = np.concatenate([[0], p, [0]])\n            runs = np.where(p[1:] != p[:-1])[0] + 1\n            runs[1::2] -= runs[::2]\n            encs.append(' '.join(str(x) for x in runs))\n    return encs\n\nclass HuBMAPDataset(Dataset):\n    def __init__(self, img_id, output_size=256, div_size=4, pad_size=72, encs=None):\n        \"\"\"\n        img_id: 画像のid\n        output_size: 出力の画像サイズ\n        div_size: 大きな画像の分割数\n        pad_size: 空白のサイズ（元画像サイズ基準）\n        \"\"\"\n        \n        self.data = rasterio.open(os.path.join(DATA, str(img_id)+'.tiff'), num_threads='all_cpus')\n        if self.data.count != 3:\n            subdatasets = self.data.subdatasets\n            self.layers = []\n            if len(subdatasets) > 0:\n                for i, subdataset in enumerate(subdatasets, 0):\n                    self.layers.append(rasterio.open(subdataset))\n                    \n                    \n        if isinstance(div_size, int):\n            div_size = (div_size, div_size)\n        if isinstance(pad_size, int):\n            pad_size = (pad_size, pad_size)\n                    \n        self.n0max, self.n1max = div_size\n        self.pad0, self.pad1 = pad_size        \n        self.shape = self.data.shape\n        self.mask = enc2mask(encs,(self.shape[1],self.shape[0])) if encs is not None else None\n        self.output_size = output_size\n        \n        # うまく分割できるように調整\n        self.pad0 += (-self.pad0 - self.shape[0]) % self.n0max\n        self.pad1 += (-self.pad0 - self.shape[1]) % self.n1max\n        \n        self.input_size0 = (self.pad0 + self.shape[0])//self.n0max\n        self.input_size1 = (self.pad1 + self.shape[1])//self.n1max\n\n    def __len__(self):\n        return self.n0max*self.n1max\n    \n    def __getitem__(self, idx):\n        # (n0, n1): 切った後の画像の位置\n        n0, n1 = idx//self.n1max, idx%self.n1max\n        # (x0, y0): 画像の左上の位置\n        x0,y0 = -self.pad0//2 + n0*self.input_size0, -self.pad1//2 + n1*self.input_size1\n\n        # make sure that the region to read is within the image\n        p00,p01 = max(0,x0), min(x0+self.input_size0,self.shape[0])\n        p10,p11 = max(0,y0), min(y0+self.input_size1,self.shape[1])\n        img = np.zeros((self.input_size0,self.input_size1,3), np.uint8)\n        mask = np.zeros((self.input_size0,self.input_size1), np.uint8)\n        \n        # mapping the loade region to the tile\n        if self.data.count == 3:\n            img[(p00-x0):(p01-x0),(p10-y0):(p11-y0)] = np.moveaxis(self.data.read([1,2,3],\n                window=Window.from_slices((p00,p01),(p10,p11))), 0, -1)\n        else:\n            for i,layer in enumerate(self.layers):\n                img[(p00-x0):(p01-x0),(p10-y0):(p11-y0),i] =\\\n                  layer.read(1,window=Window.from_slices((p00,p01),(p10,p11)))\n                \n        if self.mask is not None: mask[(p00-x0):(p01-x0),(p10-y0):(p11-y0)] = self.mask[p00:p01,p10:p11]\n        \n        img = cv2.resize(img,(self.output_size,self.output_size),\n                         interpolation = cv2.INTER_AREA)\n        mask = cv2.resize(mask,(self.output_size, self.output_size),\n                         interpolation = cv2.INTER_NEAREST)\n\n        return img, mask, idx","metadata":{"papermill":{"duration":0.028805,"end_time":"2022-08-26T00:59:29.704208","exception":false,"start_time":"2022-08-26T00:59:29.675403","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-26T08:35:35.250246Z","iopub.execute_input":"2022-08-26T08:35:35.250698Z","iopub.status.idle":"2022-08-26T08:35:35.279628Z","shell.execute_reply.started":"2022-08-26T08:35:35.250653Z","shell.execute_reply":"2022-08-26T08:35:35.277944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 指定された切り方, サイズのデータセットを作り, zipfileを出力\n\n### Setting ###\nOUTPUT_SIZE = 256\nDIV_SIZE = 3\n\nmask_map = dict(\n    kidney=1,\n    prostate=2,\n    largeintestine=3,\n    spleen=4,\n    lung=5)#add\n\n# これは基本変更しなくて良い\npad_size = 72\n###############\n\n# Outputディレクトリにすでにzipファイルが存在したら消去する\nif os.path.exists(OUT_TRAIN):\n    os.remove(OUT_TRAIN)\nif os.path.exists(OUT_MASKS):\n    os.remove(OUT_MASKS)\n    \nwith zipfile.ZipFile(OUT_TRAIN, 'w') as img_out,\\\n zipfile.ZipFile(OUT_MASKS, 'w') as mask_out:\n    for cnt, (img_id, (organ,encs)) in tqdm(enumerate(df_masks.iterrows()), total=len(df_masks)):\n        #image+mask dataset\n        ds = HuBMAPDataset(img_id, \n                           output_size=OUTPUT_SIZE, \n                           div_size=DIV_SIZE, \n                           pad_size=pad_size, \n                           encs=[encs])\n        for i in range(len(ds)): \n            img, mask, _ = ds[i]\n            mask = mask_map[organ]*mask\n            \n            # 1枚目のカットを出力して確認\n            if cnt == 0:\n                plt.title(f\"index={i}, {img.shape}, {mask.shape}\")\n                plt.imshow(img)\n                for organ_i, id in mask_map.items():\n                    print(organ_i, id, (mask==id).sum())\n                    plt.imshow((mask==id).astype(float), alpha=0.3, label=organ)\n    #                 plt.imshow(mask, alpha=0.3)\n                plt.show()\n\n            #write data   \n            img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n            img_out.writestr(f'{img_id}_{i:04d}.png', img)\n            mask = cv2.imencode('.png',mask)[1]\n            mask_out.writestr(f'{img_id}_{i:04d}.png', mask)","metadata":{"papermill":{"duration":193.091392,"end_time":"2022-08-26T01:02:42.798594","exception":false,"start_time":"2022-08-26T00:59:29.707202","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-26T08:35:35.281577Z","iopub.execute_input":"2022-08-26T08:35:35.282173Z","iopub.status.idle":"2022-08-26T08:39:18.030134Z","shell.execute_reply.started":"2022-08-26T08:35:35.282116Z","shell.execute_reply":"2022-08-26T08:39:18.027705Z"},"trusted":true},"execution_count":null,"outputs":[]}]}