{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Refrence\nMaybe the two important things in this competition are how to modeling five categories of organs and tackle size problem.\n- [HuBMAP+HPA: Data Structuring + Visualization](https://www.kaggle.com/code/girinchutia01/hubmap-hpa-data-structuring-visualization)\n- [EDA+image_processing+visualization](https://www.kaggle.com/code/arunpurakkatt/eda-image-processing-visualization)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-30T11:10:43.003039Z","iopub.execute_input":"2022-07-30T11:10:43.003398Z","iopub.status.idle":"2022-07-30T11:10:43.009048Z","shell.execute_reply.started":"2022-07-30T11:10:43.003368Z","shell.execute_reply":"2022-07-30T11:10:43.007831Z"}}},{"cell_type":"code","source":"import glob,cv2,os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:38:11.355781Z","iopub.execute_input":"2022-07-30T12:38:11.356232Z","iopub.status.idle":"2022-07-30T12:38:11.958050Z","shell.execute_reply.started":"2022-07-30T12:38:11.356196Z","shell.execute_reply":"2022-07-30T12:38:11.956866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Restructing","metadata":{}},{"cell_type":"code","source":"#explore the data of images and train_annotations\ntrain_images = sorted(glob.glob('../input/hubmap-organ-segmentation/train_images/*'))\ntrain_annotations = sorted(glob.glob('../input/hubmap-organ-segmentation/train_annotations/*'))\n\nprint(f'Total Train Annotations : {len(train_annotations)}')\nprint(f'Total Train Images : {len(train_images)}')\n\ntrain_df = pd.DataFrame.from_dict({'id' : [os.path.splitext(os.path.basename(i))[0] for i in train_images],\n                                   'Train Images Name' : [os.path.basename(i) for i in train_images], \n                                   'Train Annotations Name' : [os.path.basename(i) for i in train_annotations],\n                                  'Train Images Path' : train_images, \n                                   'Train Annotations Path' : train_annotations})\n\ntrain_df['id'] = train_df['id'].astype('int64')\ntrain_df.sample(10)","metadata":{"execution":{"iopub.status.busy":"2022-07-30T11:10:45.744628Z","iopub.execute_input":"2022-07-30T11:10:45.745008Z","iopub.status.idle":"2022-07-30T11:10:45.772733Z","shell.execute_reply.started":"2022-07-30T11:10:45.744978Z","shell.execute_reply":"2022-07-30T11:10:45.771863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# explore the data of csv\ntrain_csv = pd.read_csv('../input/hubmap-organ-segmentation/train.csv')\nprint(train_csv.shape)\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T11:10:48.522185Z","iopub.execute_input":"2022-07-30T11:10:48.522911Z","iopub.status.idle":"2022-07-30T11:10:48.709781Z","shell.execute_reply.started":"2022-07-30T11:10:48.522873Z","shell.execute_reply":"2022-07-30T11:10:48.708683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.merge(train_df, train_csv, on='id')\ntrain_csv","metadata":{"execution":{"iopub.status.busy":"2022-07-30T11:45:53.363438Z","iopub.execute_input":"2022-07-30T11:45:53.363794Z","iopub.status.idle":"2022-07-30T11:45:53.402670Z","shell.execute_reply.started":"2022-07-30T11:45:53.363767Z","shell.execute_reply":"2022-07-30T11:45:53.401666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.organ.unique()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T11:10:52.067986Z","iopub.execute_input":"2022-07-30T11:10:52.068375Z","iopub.status.idle":"2022-07-30T11:10:52.075435Z","shell.execute_reply.started":"2022-07-30T11:10:52.068344Z","shell.execute_reply":"2022-07-30T11:10:52.074549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization","metadata":{}},{"cell_type":"code","source":"def rle2mask(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n    ref : https://www.kaggle.com/paulorzp/rle-functions-run-lenght-encode-decode\n    '''\n    s = mask_rle.split()\n    starts, lengths = [\n        np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])\n    ]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = 1\n    return img.reshape(shape).T\n\ndef visualize_annotations(dataframe,organ='any'):\n    if organ != 'any':\n        dataframe = dataframe[dataframe['organ'] == organ]\n    data = dataframe.sample(6)\n    plt.figure(figsize=(15,15))\n    \n    for i in range(6):\n        plt.subplot(2,3,i+1)\n        msk = rle2mask(dataframe.iloc[i].rle,(dataframe.iloc[i].img_height,dataframe.iloc[i].img_width))\n        img = cv2.imread(dataframe.iloc[i]['Train Images Path'])\n        plt.imshow(img)\n        plt.imshow(msk,cmap='gray',alpha=0.4)\n        plt.title(dataframe.iloc[i].organ)\n        plt.axis('off')\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:15:09.495225Z","iopub.execute_input":"2022-07-30T12:15:09.495597Z","iopub.status.idle":"2022-07-30T12:15:09.510409Z","shell.execute_reply.started":"2022-07-30T12:15:09.495567Z","shell.execute_reply":"2022-07-30T12:15:09.509311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The whole distribution of data\n    data balance","metadata":{}},{"cell_type":"code","source":"sns.countplot(train_csv['organ'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:38:31.428418Z","iopub.execute_input":"2022-07-30T12:38:31.428801Z","iopub.status.idle":"2022-07-30T12:38:31.607911Z","shell.execute_reply.started":"2022-07-30T12:38:31.428768Z","shell.execute_reply":"2022-07-30T12:38:31.606676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv['organ'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:41:10.081865Z","iopub.execute_input":"2022-07-30T12:41:10.082239Z","iopub.status.idle":"2022-07-30T12:41:10.092467Z","shell.execute_reply.started":"2022-07-30T12:41:10.082209Z","shell.execute_reply":"2022-07-30T12:41:10.091022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## There are 5 categories of organs","metadata":{}},{"cell_type":"markdown","source":"### samples of prostate","metadata":{}},{"cell_type":"code","source":"visualize_annotations(train_csv,'prostate')# random sample 6 images","metadata":{"execution":{"iopub.status.busy":"2022-07-30T11:49:34.156192Z","iopub.execute_input":"2022-07-30T11:49:34.156595Z","iopub.status.idle":"2022-07-30T11:49:47.153253Z","shell.execute_reply.started":"2022-07-30T11:49:34.156563Z","shell.execute_reply":"2022-07-30T11:49:47.152110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### image size of prostate","metadata":{}},{"cell_type":"code","source":"organs=train_csv.organ.unique()\norgan=organs[0]\ntrain_pro=train_csv[train_csv.organ == organ].reset_index(drop=True)\ntrain_pro['img_width'].value_counts()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:46:44.569984Z","iopub.execute_input":"2022-07-30T12:46:44.570365Z","iopub.status.idle":"2022-07-30T12:46:44.584048Z","shell.execute_reply.started":"2022-07-30T12:46:44.570322Z","shell.execute_reply":"2022-07-30T12:46:44.582200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### samples of spleen","metadata":{}},{"cell_type":"code","source":"visualize_annotations(train_csv,'spleen')# random sample 6 images","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:40:30.718901Z","iopub.execute_input":"2022-07-30T12:40:30.719261Z","iopub.status.idle":"2022-07-30T12:40:42.802318Z","shell.execute_reply.started":"2022-07-30T12:40:30.719231Z","shell.execute_reply":"2022-07-30T12:40:42.801387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### image size of spleen","metadata":{}},{"cell_type":"code","source":"organ=organs[1]\ntrain_spleen=train_csv[train_csv.organ == organ].reset_index(drop=True)\ntrain_spleen['img_width'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:48:36.501787Z","iopub.execute_input":"2022-07-30T12:48:36.502330Z","iopub.status.idle":"2022-07-30T12:48:36.518581Z","shell.execute_reply.started":"2022-07-30T12:48:36.502285Z","shell.execute_reply":"2022-07-30T12:48:36.517337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### samples of lung","metadata":{}},{"cell_type":"code","source":"visualize_annotations(train_csv,'lung')# random sample 6 images","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:03:42.772713Z","iopub.execute_input":"2022-07-30T12:03:42.773108Z","iopub.status.idle":"2022-07-30T12:03:57.226254Z","shell.execute_reply.started":"2022-07-30T12:03:42.773077Z","shell.execute_reply":"2022-07-30T12:03:57.225475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### image size of lung","metadata":{}},{"cell_type":"code","source":"organ=organs[2]\ntrain_lung=train_csv[train_csv.organ == organ].reset_index(drop=True)\ntrain_lung['img_width'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:49:54.518633Z","iopub.execute_input":"2022-07-30T12:49:54.519005Z","iopub.status.idle":"2022-07-30T12:49:54.529448Z","shell.execute_reply.started":"2022-07-30T12:49:54.518975Z","shell.execute_reply":"2022-07-30T12:49:54.528277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### samples of kidney ","metadata":{}},{"cell_type":"code","source":"visualize_annotations(train_csv,'kidney')# random sample 6 images","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:15:16.290543Z","iopub.execute_input":"2022-07-30T12:15:16.290927Z","iopub.status.idle":"2022-07-30T12:15:30.615324Z","shell.execute_reply.started":"2022-07-30T12:15:16.290896Z","shell.execute_reply":"2022-07-30T12:15:30.614409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### image size of kidney ","metadata":{}},{"cell_type":"code","source":"organ=organs[3]\ntrain_kidney=train_csv[train_csv.organ == organ].reset_index(drop=True)\ntrain_kidney['img_width'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:50:57.301140Z","iopub.execute_input":"2022-07-30T12:50:57.301540Z","iopub.status.idle":"2022-07-30T12:50:57.315277Z","shell.execute_reply.started":"2022-07-30T12:50:57.301508Z","shell.execute_reply":"2022-07-30T12:50:57.314003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### samples of largeintestine","metadata":{}},{"cell_type":"code","source":"visualize_annotations(train_csv,'largeintestine')# random sample 6 images","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:17:44.931951Z","iopub.execute_input":"2022-07-30T12:17:44.932345Z","iopub.status.idle":"2022-07-30T12:17:59.128860Z","shell.execute_reply.started":"2022-07-30T12:17:44.932310Z","shell.execute_reply":"2022-07-30T12:17:59.127591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### image size of largeintestine","metadata":{}},{"cell_type":"code","source":"organ=organs[4]\ntrain_kidney=train_csv[train_csv.organ == organ].reset_index(drop=True)\ntrain_kidney['img_width'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-30T12:52:41.530027Z","iopub.execute_input":"2022-07-30T12:52:41.530526Z","iopub.status.idle":"2022-07-30T12:52:41.542002Z","shell.execute_reply.started":"2022-07-30T12:52:41.530491Z","shell.execute_reply":"2022-07-30T12:52:41.540838Z"},"trusted":true},"execution_count":null,"outputs":[]}]}