{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Previous Analysis　前回の分析**<br>\nhttps://www.kaggle.com/code/utm529fg/eng-hubmap-hpa-analysis-ftu-count-size","metadata":{}},{"cell_type":"markdown","source":"# 0. Prepare 準備","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import os\nimport cv2\nimport tifffile\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n#from PIL import Image\n#from PIL import ImageDraw\n\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-09-11T05:39:38.839577Z","iopub.execute_input":"2022-09-11T05:39:38.840129Z","iopub.status.idle":"2022-09-11T05:39:40.708730Z","shell.execute_reply.started":"2022-09-11T05:39:38.840013Z","shell.execute_reply":"2022-09-11T05:39:40.707551Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/code/pestipeti/decoding-rle-masks/notebook\ndef mask2rle(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\n\ndef rle2mask(mask_rle, shape=(3000,3000)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0::2], s[1::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.int32)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T","metadata":{"execution":{"iopub.status.busy":"2022-09-11T05:40:43.797009Z","iopub.execute_input":"2022-09-11T05:40:43.797497Z","iopub.status.idle":"2022-09-11T05:40:43.809157Z","shell.execute_reply.started":"2022-09-11T05:40:43.797458Z","shell.execute_reply":"2022-09-11T05:40:43.807692Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_image(idx):\n    fig, ax = plt.subplots(1,3,figsize=(12,12))\n    image = plt.imread(f\"../input/hubmap-organ-segmentation/train_images/{df_train.id[idx]}.tiff\")\n    mask = rle2mask(df_train.rle[idx],shape=(df_train.img_height[idx],df_train.img_width[idx]))\n    display(pd.DataFrame(df_train.loc[idx,['id','organ','age','sex']]).T)\n    ax[0].imshow(image)\n    ax[0].set_title(\"biopsy slide\")\n    ax[0].axis(\"off\")\n    ax[1].imshow(mask,alpha=0.3,cmap='gray')\n    ax[1].set_title(\"FTUs mask\")\n    ax[1].axis(\"off\")\n    ax[2].imshow(image)\n    ax[2].imshow(mask,alpha=0.3,cmap='gray')\n    ax[2].axis(\"off\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-11T05:40:55.802149Z","iopub.execute_input":"2022-09-11T05:40:55.802652Z","iopub.status.idle":"2022-09-11T05:40:55.812444Z","shell.execute_reply.started":"2022-09-11T05:40:55.802609Z","shell.execute_reply":"2022-09-11T05:40:55.811311Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/hubmap-organ-segmentation/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-09-11T05:40:40.877051Z","iopub.execute_input":"2022-09-11T05:40:40.878432Z","iopub.status.idle":"2022-09-11T05:40:41.197879Z","shell.execute_reply.started":"2022-09-11T05:40:40.878378Z","shell.execute_reply":"2022-09-11T05:40:41.196385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Calculate Size and Count of FTUs サイズとカウントの計算","metadata":{}},{"cell_type":"code","source":"def fill_1mask(arr):\n    if arr.max()==0:\n        return\n    else:\n        y = np.where(arr>0)[0][0]\n        x = np.where(arr>0)[1][0]\n        h, w = arr.shape\n        mask = np.zeros((h+2, w+2), np.uint8)\n        retval,rimage,mask,rect = cv2.floodFill(cv2.UMat(arr), mask, (x,y), 0,flags=8)\n        return rimage.get()\n\ndef show_mask_per_one(idx):\n    arr = rle2mask(df_train.rle[idx],shape=(df_train.img_height[idx],df_train.img_width[idx]))\n    fig, ax = plt.subplots(1,1,figsize=(4,4))\n    ax.set_title('All FTUs')\n    ax.axis(\"off\")\n    ax.imshow(arr,cmap='gray')\n    \n    while arr.max()>0:\n        tmp = fill_1mask(arr)\n        fig, ax = plt.subplots(1,1,figsize=(4,4))\n        ax.imshow(arr - tmp,cmap='gray')\n        ax.set_title('pixel size {}'.format(sum(sum(arr - tmp))))\n        ax.axis(\"off\")\n        arr = tmp\n\ndef get_mask_size_per_one(idx):\n    arr = rle2mask(df_train.rle[idx],shape=(df_train.img_height[idx],df_train.img_width[idx]))\n    mask_size=[]\n    while arr.max()>0:\n        tmp = fill_1mask(arr)\n        mask_size.append(sum(sum(arr - tmp)))\n        arr = tmp\n    return(np.array(mask_size))","metadata":{"execution":{"iopub.status.busy":"2022-09-11T05:42:42.819299Z","iopub.execute_input":"2022-09-11T05:42:42.822416Z","iopub.status.idle":"2022-09-11T05:42:42.844602Z","shell.execute_reply.started":"2022-09-11T05:42:42.822296Z","shell.execute_reply":"2022-09-11T05:42:42.842746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx in tqdm(range(len(df_train))):\n    tmp_mask_size = get_mask_size_per_one(idx)\n    tmp_mask_size = np.concatenate([[df_train.id[idx]]*len(tmp_mask_size), tmp_mask_size])\n    tmp_mask_size = np.reshape(tmp_mask_size,(2,int(len(tmp_mask_size)/2))).T\n    if idx == 0:\n        mask_size = tmp_mask_size.copy()\n    else:\n        mask_size = np.concatenate([mask_size, tmp_mask_size])","metadata":{"execution":{"iopub.status.busy":"2022-09-11T05:42:57.500015Z","iopub.execute_input":"2022-09-11T05:42:57.500445Z","iopub.status.idle":"2022-09-11T05:53:56.482216Z","shell.execute_reply.started":"2022-09-11T05:42:57.500411Z","shell.execute_reply":"2022-09-11T05:53:56.481011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_size = pd.merge(df_train,\n                         pd.DataFrame(mask_size,columns=['id','FTU_pixel_size']),\n                         on='id',how='outer')\n\n# 画像ピクセルサイズに対するFTUのピクセルサイズの割合\ndf_train_size['FTU_size_rate'] = df_train_size['FTU_pixel_size']/(df_train_size['img_height']*df_train_size['img_width'])","metadata":{"execution":{"iopub.status.busy":"2022-09-11T05:53:56.484468Z","iopub.execute_input":"2022-09-11T05:53:56.484843Z","iopub.status.idle":"2022-09-11T05:53:56.527620Z","shell.execute_reply.started":"2022-09-11T05:53:56.484809Z","shell.execute_reply":"2022-09-11T05:53:56.526490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_size.loc[:,['id','organ','age','sex','FTU_pixel_size','FTU_size_rate']].sample(10)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T05:56:32.809857Z","iopub.execute_input":"2022-09-11T05:56:32.810632Z","iopub.status.idle":"2022-09-11T05:56:32.830579Z","shell.execute_reply.started":"2022-09-11T05:56:32.810556Z","shell.execute_reply":"2022-09-11T05:56:32.829530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Extra Small FTU 極小FTU","metadata":{}},{"cell_type":"code","source":"df_extra_small_ftu = df_train_size.loc[:,['id','organ','age','sex','FTU_pixel_size','FTU_size_rate']].sort_values('FTU_size_rate').drop_duplicates(subset='id').head(20)\ndf_extra_small_ftu","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:25:30.067992Z","iopub.execute_input":"2022-09-11T07:25:30.068466Z","iopub.status.idle":"2022-09-11T07:25:30.095124Z","shell.execute_reply.started":"2022-09-11T07:25:30.068430Z","shell.execute_reply":"2022-09-11T07:25:30.093865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check for FTU size lower 20<br>\nFTUのサイズ下位20について確認する","metadata":{}},{"cell_type":"code","source":"def show_mask_per_one_min(idx,xs,xe,ys,ye):\n    fig, ax = plt.subplots(1,3,figsize=(20,40))\n    image = plt.imread(f\"../input/hubmap-organ-segmentation/train_images/{df_train.id[idx]}.tiff\")\n    arr = rle2mask(df_train.rle[idx],shape=(df_train.img_height[idx],df_train.img_width[idx]))\n    \n    image = image[xs:xe,ys:ye]\n    arr = arr[xs:xe,ys:ye]\n    \n    display(pd.DataFrame(df_train.loc[idx,['id','organ','age','sex']]).T)\n    ax[0].imshow(image)\n    ax[0].set_title(\"Biopsy slide\")\n    ax[0].axis(\"off\")\n    \n    ax[1].set_title('All FTUs')\n    ax[1].axis(\"off\")\n    ax[1].imshow(arr,cmap='plasma')\n    \n    ax[2].imshow(image)\n    ax[2].imshow(arr,cmap='plasma',alpha = 0.5)\n    ax[2].set_title('Biopsy slide with FTU mask')\n    ax[2].axis(\"off\")\n\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:28:07.868427Z","iopub.execute_input":"2022-09-11T06:28:07.869226Z","iopub.status.idle":"2022-09-11T06:28:07.879884Z","shell.execute_reply.started":"2022-09-11T06:28:07.869171Z","shell.execute_reply":"2022-09-11T06:28:07.878572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 0\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1900,2100,1500,1700)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:42:53.649005Z","iopub.execute_input":"2022-09-11T06:42:53.650334Z","iopub.status.idle":"2022-09-11T06:42:54.135642Z","shell.execute_reply.started":"2022-09-11T06:42:53.650279Z","shell.execute_reply":"2022-09-11T06:42:54.134503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 9387 extra small FTU is mislabeled.<br>\n明らかに誤ったラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 1\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1900,2300,1800,2200)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:43:23.751765Z","iopub.execute_input":"2022-09-11T06:43:23.753262Z","iopub.status.idle":"2022-09-11T06:43:24.366247Z","shell.execute_reply.started":"2022-09-11T06:43:23.753216Z","shell.execute_reply":"2022-09-11T06:43:24.365008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 26480 extra small FTU is probably mislabeled between two FTUs.<br>\n2つのFTUの間にあるのは、おそらく誤ったラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 2\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1700,2200,1000,1500)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:45:04.527554Z","iopub.execute_input":"2022-09-11T06:45:04.528082Z","iopub.status.idle":"2022-09-11T06:45:05.112401Z","shell.execute_reply.started":"2022-09-11T06:45:04.528041Z","shell.execute_reply":"2022-09-11T06:45:05.111291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 25298 extra small FTU is probably mislabeled.<br>\nおそらく誤ったラベルである。","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:45:36.725045Z","iopub.execute_input":"2022-09-11T06:45:36.725517Z","iopub.status.idle":"2022-09-11T06:45:36.733280Z","shell.execute_reply.started":"2022-09-11T06:45:36.725482Z","shell.execute_reply":"2022-09-11T06:45:36.731655Z"}}},{"cell_type":"code","source":"idx = 3\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1500,1700,200,400)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:49:25.334857Z","iopub.execute_input":"2022-09-11T06:49:25.335301Z","iopub.status.idle":"2022-09-11T06:49:25.770546Z","shell.execute_reply.started":"2022-09-11T06:49:25.335261Z","shell.execute_reply":"2022-09-11T06:49:25.769667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 15499 extra small FTU is probably mislabeled.<br>\nおそらく誤ったラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 4\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1800,2200,1000,1400)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:51:34.257335Z","iopub.execute_input":"2022-09-11T06:51:34.258057Z","iopub.status.idle":"2022-09-11T06:51:34.777248Z","shell.execute_reply.started":"2022-09-11T06:51:34.258003Z","shell.execute_reply":"2022-09-11T06:51:34.775980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 4301 extra small FTU is mislabeled.<br>\n明らかに誤ったラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 5\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1200,1400,1600,1800)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:55:11.018659Z","iopub.execute_input":"2022-09-11T06:55:11.019102Z","iopub.status.idle":"2022-09-11T06:55:11.602509Z","shell.execute_reply.started":"2022-09-11T06:55:11.019063Z","shell.execute_reply":"2022-09-11T06:55:11.601420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 6611 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 6\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1300,1600,500,800)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:58:35.689144Z","iopub.execute_input":"2022-09-11T06:58:35.689607Z","iopub.status.idle":"2022-09-11T06:58:36.221306Z","shell.execute_reply.started":"2022-09-11T06:58:35.689557Z","shell.execute_reply":"2022-09-11T06:58:36.220244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 1850 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{"execution":{"iopub.status.busy":"2022-09-11T06:58:55.149814Z","iopub.execute_input":"2022-09-11T06:58:55.150776Z","iopub.status.idle":"2022-09-11T06:58:55.158292Z","shell.execute_reply.started":"2022-09-11T06:58:55.150727Z","shell.execute_reply":"2022-09-11T06:58:55.157122Z"}}},{"cell_type":"code","source":"idx = 7\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],2200,2700,300,800)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:00:38.800027Z","iopub.execute_input":"2022-09-11T07:00:38.801304Z","iopub.status.idle":"2022-09-11T07:00:39.411403Z","shell.execute_reply.started":"2022-09-11T07:00:38.801241Z","shell.execute_reply":"2022-09-11T07:00:39.410043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 10892 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 8\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],300,600,1200,1500)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:02:46.950135Z","iopub.execute_input":"2022-09-11T07:02:46.950620Z","iopub.status.idle":"2022-09-11T07:02:47.504472Z","shell.execute_reply.started":"2022-09-11T07:02:46.950560Z","shell.execute_reply":"2022-09-11T07:02:47.503106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 17828 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:03:01.369121Z","iopub.execute_input":"2022-09-11T07:03:01.369571Z","iopub.status.idle":"2022-09-11T07:03:01.376646Z","shell.execute_reply.started":"2022-09-11T07:03:01.369532Z","shell.execute_reply":"2022-09-11T07:03:01.375054Z"}}},{"cell_type":"code","source":"idx = 9\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],0,1000,0,1000)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:04:43.770159Z","iopub.execute_input":"2022-09-11T07:04:43.770621Z","iopub.status.idle":"2022-09-11T07:04:44.678482Z","shell.execute_reply.started":"2022-09-11T07:04:43.770564Z","shell.execute_reply":"2022-09-11T07:04:44.677360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 18445 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 10\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],700,1200,2400,2900)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:07:08.452478Z","iopub.execute_input":"2022-09-11T07:07:08.452995Z","iopub.status.idle":"2022-09-11T07:07:09.056150Z","shell.execute_reply.started":"2022-09-11T07:07:08.452953Z","shell.execute_reply":"2022-09-11T07:07:09.054458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 4062 perhaps has broken labels.<br>\nおそらくラベルが途中で途切れている。","metadata":{}},{"cell_type":"code","source":"idx = 11\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],600,1100,1200,1700)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:12:32.995264Z","iopub.execute_input":"2022-09-11T07:12:32.995779Z","iopub.status.idle":"2022-09-11T07:12:33.576036Z","shell.execute_reply.started":"2022-09-11T07:12:32.995736Z","shell.execute_reply":"2022-09-11T07:12:33.574826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 8222 extra small FTU is the correct label. But the label is a bit broken.<br>\n正しいラベルである。ただ、少し形が崩れている。","metadata":{}},{"cell_type":"code","source":"idx = 12\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1300,1800,1900,2400)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:18:17.155367Z","iopub.execute_input":"2022-09-11T07:18:17.155832Z","iopub.status.idle":"2022-09-11T07:18:17.702128Z","shell.execute_reply.started":"2022-09-11T07:18:17.155791Z","shell.execute_reply":"2022-09-11T07:18:17.700882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 12471 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 13\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1000,1500,1000,1500)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:19:52.198654Z","iopub.execute_input":"2022-09-11T07:19:52.199122Z","iopub.status.idle":"2022-09-11T07:19:52.745992Z","shell.execute_reply.started":"2022-09-11T07:19:52.199084Z","shell.execute_reply":"2022-09-11T07:19:52.744410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 27128 extra small FTU perhaps split in two.<br>\nおそらくラベルが2つに分裂している。","metadata":{}},{"cell_type":"code","source":"idx = 14\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],700,1200,1500,2000)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:23:29.329628Z","iopub.execute_input":"2022-09-11T07:23:29.330460Z","iopub.status.idle":"2022-09-11T07:23:29.874314Z","shell.execute_reply.started":"2022-09-11T07:23:29.330414Z","shell.execute_reply":"2022-09-11T07:23:29.873093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 26319 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 15\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],2200,2700,1500,2000)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:26:24.192355Z","iopub.execute_input":"2022-09-11T07:26:24.193332Z","iopub.status.idle":"2022-09-11T07:26:24.812945Z","shell.execute_reply.started":"2022-09-11T07:26:24.193283Z","shell.execute_reply":"2022-09-11T07:26:24.811431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 22016 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 16\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1500,2000,1500,2000)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:27:42.306436Z","iopub.execute_input":"2022-09-11T07:27:42.306868Z","iopub.status.idle":"2022-09-11T07:27:42.902833Z","shell.execute_reply.started":"2022-09-11T07:27:42.306831Z","shell.execute_reply":"2022-09-11T07:27:42.901393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 11890 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 17\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],2000,3000,600,1600)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:29:05.488659Z","iopub.execute_input":"2022-09-11T07:29:05.489537Z","iopub.status.idle":"2022-09-11T07:29:06.400585Z","shell.execute_reply.started":"2022-09-11T07:29:05.489482Z","shell.execute_reply":"2022-09-11T07:29:06.399463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 203 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 18\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],800,1700,1900,2800)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:31:37.478301Z","iopub.execute_input":"2022-09-11T07:31:37.479411Z","iopub.status.idle":"2022-09-11T07:31:38.269959Z","shell.execute_reply.started":"2022-09-11T07:31:37.479362Z","shell.execute_reply":"2022-09-11T07:31:38.268762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 25641 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{}},{"cell_type":"code","source":"idx = 19\nshow_mask_per_one_min(df_train[df_train['id']==df_extra_small_ftu.iloc[idx,0]].index[0],1500,2500,800,1800)","metadata":{"execution":{"iopub.status.busy":"2022-09-11T07:32:35.301708Z","iopub.execute_input":"2022-09-11T07:32:35.302133Z","iopub.status.idle":"2022-09-11T07:32:36.188024Z","shell.execute_reply.started":"2022-09-11T07:32:35.302100Z","shell.execute_reply":"2022-09-11T07:32:36.186976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Id 21501 extra small FTU is the correct label.<br>\n正しいラベルである。","metadata":{}},{"cell_type":"markdown","source":"# 3. Consideration 考察","metadata":{}},{"cell_type":"markdown","source":"- All FTUs with pixel size less than 200 are incorrect. So we should be removed from the training target and prediction results.<br>\nピクセルサイズ200未満のFTUは全て誤り。学習・予測からは削除した方が良さそう。\n- Even with pixel sizes of 1000 or more, there are clearly erroneous labels. However, very small FTUs also exist. If we can correctly train and predict here, the accuracy will be improved.<br>\nピクセルサイズ1000以上でも、明らかに誤りのラベルが混在する。ここを正しく学習・予測できれば精度向上するかも。\n- Incorrect data found in this notebook are as follows:<br>今回見つけた誤りデータはこちら<br>\nNot FTU : ID 9387, 26480, 25298, 15499 and 4301<br>\nBroken mask : ID 4062 and 27128","metadata":{}}]}