{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport cv2\nimport tifffile as tiff \nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom glob import glob\nfrom PIL import Image, ImageOps\nfrom sklearn.metrics import f1_score, roc_auc_score\n\nimport torch\nimport torch.nn as nn\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import AdamW, lr_scheduler\nfrom torchvision import transforms","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-17T13:04:18.335782Z","iopub.execute_input":"2022-07-17T13:04:18.336773Z","iopub.status.idle":"2022-07-17T13:04:22.374296Z","shell.execute_reply.started":"2022-07-17T13:04:18.336651Z","shell.execute_reply":"2022-07-17T13:04:22.373046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploratory data analysis","metadata":{}},{"cell_type":"code","source":"path = '../input/hubmap-organ-segmentation/'\nimg_path = path + 'train_images/'","metadata":{"execution":{"iopub.status.busy":"2022-07-17T13:04:22.376191Z","iopub.execute_input":"2022-07-17T13:04:22.377676Z","iopub.status.idle":"2022-07-17T13:04:22.382502Z","shell.execute_reply.started":"2022-07-17T13:04:22.377616Z","shell.execute_reply":"2022-07-17T13:04:22.381286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(path + 'train.csv')\ntest_df = pd.read_csv(path + 'test.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T13:04:22.384047Z","iopub.execute_input":"2022-07-17T13:04:22.384658Z","iopub.status.idle":"2022-07-17T13:04:22.776251Z","shell.execute_reply.started":"2022-07-17T13:04:22.384622Z","shell.execute_reply":"2022-07-17T13:04:22.775155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The dataset is mostly kidney and prostate, which is concerning since the public test organ is spleen, the second least represented organ. The distribution of males is twice females, explaining the large amount of prostate images. I'm not sure how important the gender imbalance is for semantic segmentation. Age is unimodal at 60 with gaps between 30-35 and 50-55.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(24, 8))\n\nsns.countplot(data=df, x='organ', ax=ax[0])\nsns.countplot(data=df, x='sex', ax=ax[1])\nsns.histplot(data=df, x='age', ax=ax[2])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T13:04:22.778396Z","iopub.execute_input":"2022-07-17T13:04:22.778763Z","iopub.status.idle":"2022-07-17T13:04:23.391942Z","shell.execute_reply.started":"2022-07-17T13:04:22.778733Z","shell.execute_reply":"2022-07-17T13:04:23.390677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode(img):\n    pixels = img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef rle_decode(mask_rle, shape=(3000, 3000), get_stat=False):\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    if get_stat:\n        return img.sum()\n    return img.reshape(shape).T","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-17T13:04:23.393312Z","iopub.execute_input":"2022-07-17T13:04:23.393660Z","iopub.status.idle":"2022-07-17T13:04:23.403480Z","shell.execute_reply.started":"2022-07-17T13:04:23.393629Z","shell.execute_reply":"2022-07-17T13:04:23.402132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['pixel_count'] = df.apply(lambda x: rle_decode(x['rle'], shape=(x['img_width'], x['img_height']), get_stat=True), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T13:04:23.405188Z","iopub.execute_input":"2022-07-17T13:04:23.405537Z","iopub.status.idle":"2022-07-17T13:04:28.709487Z","shell.execute_reply.started":"2022-07-17T13:04:23.405505Z","shell.execute_reply":"2022-07-17T13:04:28.708213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['zero_count'] = df['img_width'] * df['img_height'] - df['pixel_count']\ndf['pct_empty'] = df['zero_count'] / (df['img_width'] * df['img_height'])\ndf['pct_full'] = 1 - df['pct_empty']\ndf.groupby('organ')['pct_empty'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T13:04:28.710935Z","iopub.execute_input":"2022-07-17T13:04:28.711263Z","iopub.status.idle":"2022-07-17T13:04:28.747003Z","shell.execute_reply.started":"2022-07-17T13:04:28.711232Z","shell.execute_reply":"2022-07-17T13:04:28.745964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pixel distribution of lungs and kidney is tight with prostate having a wide variance. The largeintestine for females also has a wide variance. The overall percentage of empty organs is left skewed, stating that most masks are more empty than full.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(20, 8))\nsns.boxplot(data=df, x='organ', y='pct_full', hue='sex', ax=ax[0])\nsns.histplot(data=df, x='pct_empty', ax=ax[1])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T13:04:28.748305Z","iopub.execute_input":"2022-07-17T13:04:28.749336Z","iopub.status.idle":"2022-07-17T13:04:29.236341Z","shell.execute_reply.started":"2022-07-17T13:04:28.749291Z","shell.execute_reply":"2022-07-17T13:04:29.234959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(20, 8))\nsns.violinplot(data=df, x='organ', y='pixel_count', hue='sex', ax=ax[0])\nsns.violinplot(data=df, x='organ', y='age', hue='sex', ax=ax[1])\nsns.histplot(df, hue=\"organ\", x=\"pct_full\",  multiple=\"dodge\", ax=ax[2])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T13:04:29.238082Z","iopub.execute_input":"2022-07-17T13:04:29.238419Z","iopub.status.idle":"2022-07-17T13:04:30.208024Z","shell.execute_reply.started":"2022-07-17T13:04:29.238388Z","shell.execute_reply":"2022-07-17T13:04:30.207126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def draw_bb(mask):\n    contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n    (x,y,w,h) = cv2.boundingRect(contours[0])\n    return x,y,w,h","metadata":{"execution":{"iopub.status.busy":"2022-07-17T13:14:52.468156Z","iopub.execute_input":"2022-07-17T13:14:52.468565Z","iopub.status.idle":"2022-07-17T13:14:52.473694Z","shell.execute_reply.started":"2022-07-17T13:14:52.468530Z","shell.execute_reply":"2022-07-17T13:14:52.472685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_new_images(df):\n    edf = df.sort_values(['pct_empty'], ascending=False).query('pct_empty > 0.99')[['id', 'rle', 'img_width', 'img_height']].head(20).reset_index(drop=True)\n    image = np.zeros((3000, 3000, 3), dtype='uint8')\n    mask = np.zeros((3000, 3000))\n    xl = 0\n    yt = 0\n    pad = 20\n    for idx, row in edf.iterrows(): \n        sample_image = tiff.imread(img_path + '{}.tiff'.format(row['id']))\n        sample_mask = rle_decode(row['rle'], shape=(row['img_width'], row['img_height']))\n        x,y,wi,he = draw_bb(sample_mask)\n        if xl + wi > 3000:\n            break\n            xl = 0\n            #yt = np.argmin(mask[.sum(axis=1) + 10\n            yt = np.nonzero(nmask.sum(1))[0][-1] + pad\n        if (yt > 3000) or (yt + he > 3000):\n            break\n        crop_img = sample_image[y-pad:y+he+pad, x-pad:x+wi+pad]\n        crop_mask = sample_mask[y-pad:y+he+pad, x-pad:x+wi+pad]\n        if idx > 16:\n            continue\n        image[yt:yt+he+2*pad, xl:xl+wi+2*pad] = crop_img\n        mask[yt:yt+he+2*pad, xl:xl+wi+2*pad] = crop_mask\n\n        xl += wi\n        #print('xl: {} yt: {}'.format(xl, yt))\n    return image, mask\n\nnimg, nmask = create_new_images(df)\nfig, ax = plt.subplots(1, 2)\nax[0].imshow(nimg)\nax[1].imshow(nmask)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-17T13:14:53.204867Z","iopub.execute_input":"2022-07-17T13:14:53.205617Z","iopub.status.idle":"2022-07-17T13:15:00.008934Z","shell.execute_reply.started":"2022-07-17T13:14:53.205550Z","shell.execute_reply":"2022-07-17T13:15:00.007925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1, figsize=(20, 10))\nax.imshow(nmask[:562+20], cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2022-07-17T13:15:00.010475Z","iopub.execute_input":"2022-07-17T13:15:00.010807Z","iopub.status.idle":"2022-07-17T13:15:00.374808Z","shell.execute_reply.started":"2022-07-17T13:15:00.010780Z","shell.execute_reply":"2022-07-17T13:15:00.373632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smp = 85\nname = df.loc[smp]['id']\nwidth = height = df.loc[smp]['img_height']\nsmp_image = img_path + '{}.tiff'.format(name)\nimg = tiff.imread(smp_image)\n\nrle = df.loc[smp]['rle']\nmask = rle_decode(rle, shape=(width, height))\nx,y,w,h = draw_bb(mask)\nxs = [x, x+w, x+w, x, x]; ys = [y, y, y+h, y+h, y]\n\nfig, ax = plt.subplots(1, 3, figsize=(15, 10), sharey=True)\nax[0].imshow(img)\nax[1].imshow(mask,cmap='gray')\nax[1].plot(xs, ys, color='red')\n\nax[2].imshow(img)\nax[2].imshow(mask, cmap='cool', alpha=0.5)\n\nax[2].set_title(name)\nax[2].plot(xs, ys, color='green')\nax[2].axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-07-17T13:18:40.435971Z","iopub.execute_input":"2022-07-17T13:18:40.436499Z","iopub.status.idle":"2022-07-17T13:18:43.584275Z","shell.execute_reply.started":"2022-07-17T13:18:40.436442Z","shell.execute_reply":"2022-07-17T13:18:43.579725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_grid(df):\n    sample = 8\n    df = df.sample(sample).reset_index(drop=True)\n    fig, ax = plt.subplots(2, sample, figsize=(14, 8))\n    for smp, i in enumerate(ax.flatten()):\n        if smp < sample:\n            name = df.loc[smp]['id']\n            width = height = df.loc[smp]['img_height']\n            smp_image = img_path + '{}.tiff'.format(name)\n            img = tiff.imread(smp_image)\n            i.imshow(img)\n            i.set_title(name)\n            i.axis(\"off\")\n        else:\n            pixel_count, pct_full = df.loc[smp-sample]['pixel_count'], df.loc[smp-sample]['pct_full']\n            mask = df.loc[smp-sample]['mask']\n            i.imshow(mask)\n            i.set_title('pc: {:.2f} pf: {:.2f}'.format(pixel_count/1000, pct_full))\n            i.axis(\"off\")\n    plt.tight_layout()\n    \ndef dice_score(im1, im2=False, test=None):\n    im1 = np.asarray(im1).astype(bool)\n    if test == 'full':\n        im2 = np.ones(im1.shape[:2])\n    elif test == 'empty':\n        im2 = np.zeros(im1.shape[:2])\n    else:\n        im2 = np.asarray(im2).astype(bool)\n    intersection = np.logical_and(im1, im2)\n    return 2. * intersection.sum() / (im1.sum() + im2.sum() + 1e-8)\n\ndef mask_test(df):\n    df['mask'] = df.apply(lambda x: rle_decode(x['rle'], shape=(x['img_width'], x['img_height'])), axis=1)\n    df['dice_full'] = df.apply(lambda x: dice_score(x['mask'], test='full'), axis=1)\n    df['dice_empty'] = df.apply(lambda x: dice_score(x['mask'], test='empty'), axis=1)\n    return df\n\ndef plot_hist(df):\n    fig, ax = plt.subplots(1, 3, figsize=(20, 6))\n    df['dice_full'].plot(kind='hist', ax=ax[0], title='Dice full')\n    df['pct_full'].plot(kind='hist', ax=ax[1], title='Percentage Full')\n    df['pixel_count'].plot(kind='hist', ax=ax[2], title='Pixel Count Distribution')","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:02:18.425769Z","iopub.execute_input":"2022-07-13T18:02:18.426194Z","iopub.status.idle":"2022-07-13T18:02:18.443049Z","shell.execute_reply.started":"2022-07-13T18:02:18.426160Z","shell.execute_reply":"2022-07-13T18:02:18.441567Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we take a look at each organ. For all of the analysis \n- pc: The pixel count in 1000s, so 104 is 104e3 number of pixels.\n- pf: Percentage of the image that has the positive label\n- dice full: Dice score if a mask of ones was submitted","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:51:37.086043Z","iopub.execute_input":"2022-07-13T18:51:37.086584Z","iopub.status.idle":"2022-07-13T18:51:37.097509Z","shell.execute_reply.started":"2022-07-13T18:51:37.086525Z","shell.execute_reply":"2022-07-13T18:51:37.095692Z"}}},{"cell_type":"code","source":"df = mask_test(df)\ndf.groupby('organ')['dice_full'].describe()","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prostate Analysis","metadata":{}},{"cell_type":"markdown","source":"The prostate is a walnut-sized gland located between the bladder and the penis. The prostate is just in front of the rectum. The urethra runs through the center of the prostate, from the bladder to the penis, letting urine flow out of the body. [Source](https://www.webmd.com/men/picture-of-the-prostate)","metadata":{}},{"cell_type":"code","source":"pdf = df[df['organ'].eq('prostate')]\nplot_grid(pdf)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:02:20.805123Z","iopub.execute_input":"2022-07-13T18:02:20.806584Z","iopub.status.idle":"2022-07-13T18:02:34.703477Z","shell.execute_reply.started":"2022-07-13T18:02:20.806527Z","shell.execute_reply":"2022-07-13T18:02:34.702464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(pdf)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:02:34.704902Z","iopub.execute_input":"2022-07-13T18:02:34.705252Z","iopub.status.idle":"2022-07-13T18:02:35.151281Z","shell.execute_reply.started":"2022-07-13T18:02:34.705219Z","shell.execute_reply":"2022-07-13T18:02:35.150376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For the prostate the pixel distribution and percentage full are both unimodal and right skewed","metadata":{}},{"cell_type":"markdown","source":"## Spleen Analysis","metadata":{"execution":{"iopub.status.busy":"2022-07-13T14:42:55.120090Z","iopub.execute_input":"2022-07-13T14:42:55.120488Z","iopub.status.idle":"2022-07-13T14:42:55.124900Z","shell.execute_reply.started":"2022-07-13T14:42:55.120455Z","shell.execute_reply":"2022-07-13T14:42:55.124098Z"}}},{"cell_type":"markdown","source":"The spleen is an organ in the upper far left part of the abdomen, to the left of the stomach. The spleen varies in size and shape between people, but it’s commonly fist-shaped, purple, and about 4 inches long. Because the spleen is protected by the rib cage, you can’t easily feel it unless it’s abnormally enlarged. [source](https://www.webmd.com/digestive-disorders/picture-of-the-spleen)","metadata":{}},{"cell_type":"code","source":"sdf = df[df['organ'].eq('spleen')]\nplot_grid(sdf)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:02:35.152328Z","iopub.execute_input":"2022-07-13T18:02:35.153218Z","iopub.status.idle":"2022-07-13T18:02:48.395778Z","shell.execute_reply.started":"2022-07-13T18:02:35.153180Z","shell.execute_reply":"2022-07-13T18:02:48.394655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(sdf)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:02:48.398223Z","iopub.execute_input":"2022-07-13T18:02:48.398529Z","iopub.status.idle":"2022-07-13T18:02:48.859525Z","shell.execute_reply.started":"2022-07-13T18:02:48.398502Z","shell.execute_reply":"2022-07-13T18:02:48.858246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The spleen is similar to the prostate with both pixel distribution and percentage full being right skewed. The masks for the spleen have large FTU than the very small ones of the prostate","metadata":{}},{"cell_type":"markdown","source":"## Lung Analysis","metadata":{}},{"cell_type":"markdown","source":"The lungs are a pair of spongy, air-filled organs located on either side of the chest (thorax). The trachea (windpipe) conducts inhaled air into the lungs through its tubular branches, called bronchi. The bronchi then divide into smaller and smaller branches (bronchioles), finally becoming microscopic. \n[source](https://www.webmd.com/lung/picture-of-the-lungs#:~:text=The%20lungs%20are%20a%20pair,bronchioles)","metadata":{}},{"cell_type":"code","source":"ldf = df[df['organ'].eq('lung')]\nplot_grid(ldf)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:02:48.860951Z","iopub.execute_input":"2022-07-13T18:02:48.861319Z","iopub.status.idle":"2022-07-13T18:03:03.443985Z","shell.execute_reply.started":"2022-07-13T18:02:48.861285Z","shell.execute_reply":"2022-07-13T18:03:03.442710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(ldf)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:03:03.446452Z","iopub.execute_input":"2022-07-13T18:03:03.447546Z","iopub.status.idle":"2022-07-13T18:03:03.883806Z","shell.execute_reply.started":"2022-07-13T18:03:03.447498Z","shell.execute_reply":"2022-07-13T18:03:03.882674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The masks for the lungs are very sparse. Predicting all ones is high risk and low reward. Most of the images are near empty with some images having only 32k positive pixels. A lot of the images are very jagged. The historgrams show a lot of holes in the positive labels. The percentage full is the lowest of all organs with all images in the single percentage digits. If the private test set has a lot of lung images it can reduce dice scores.","metadata":{}},{"cell_type":"markdown","source":"## Kidney Analysis","metadata":{}},{"cell_type":"markdown","source":"The kidneys are a pair of bean-shaped organs on either side of your spine, below your ribs and behind your belly. Each kidney is about 4 or 5 inches long, roughly the size of a large fist. [source](https://www.webmd.com/kidney-stones/picture-of-the-kidneys)","metadata":{}},{"cell_type":"code","source":"kdf = df[df['organ'].eq('kidney')]\nplot_grid(kdf)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:03:06.915649Z","iopub.execute_input":"2022-07-13T18:03:06.916040Z","iopub.status.idle":"2022-07-13T18:03:19.199822Z","shell.execute_reply.started":"2022-07-13T18:03:06.916001Z","shell.execute_reply":"2022-07-13T18:03:19.198661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(kdf)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:03:19.201986Z","iopub.execute_input":"2022-07-13T18:03:19.203042Z","iopub.status.idle":"2022-07-13T18:03:19.627090Z","shell.execute_reply.started":"2022-07-13T18:03:19.202995Z","shell.execute_reply":"2022-07-13T18:03:19.625685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Similar to the lungs, the kidney is also sparse; however the masks are mostly circular. The distribution is more smooth overall for the percentage full and the pixel counts.","metadata":{}},{"cell_type":"markdown","source":"## Largeintestine Analysis","metadata":{}},{"cell_type":"markdown","source":"The intestines are a long, continuous tube running from the stomach to the anus. Most absorption of nutrients and water happen in the intestines. The intestines include the small intestine, large intestine, and rectum. The large intestine (colon or large bowel) is about 5 feet long and about 3 inches in diameter. The colon absorbs water from wastes, creating stool. As stool enters the rectum, nerves there create the urge to defecate. [source](https://www.webmd.com/digestive-disorders/picture-of-the-intestines)","metadata":{}},{"cell_type":"code","source":"lidf = df[df['organ'].eq('largeintestine')]\nplot_grid(lidf)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:05:07.073855Z","iopub.execute_input":"2022-07-13T18:05:07.075262Z","iopub.status.idle":"2022-07-13T18:05:21.457575Z","shell.execute_reply.started":"2022-07-13T18:05:07.075203Z","shell.execute_reply":"2022-07-13T18:05:21.456671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist(lidf)","metadata":{"execution":{"iopub.status.busy":"2022-07-13T18:05:21.459523Z","iopub.execute_input":"2022-07-13T18:05:21.460096Z","iopub.status.idle":"2022-07-13T18:05:21.851717Z","shell.execute_reply.started":"2022-07-13T18:05:21.460060Z","shell.execute_reply":"2022-07-13T18:05:21.850868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The largeintestine is the only organ that is almost normally distributed. The masks have the most positive labels out of all organs. If the private test set has a lot of largeintestine it can inflate dice scores.","metadata":{}},{"cell_type":"markdown","source":"## Conclusion","metadata":{}},{"cell_type":"markdown","source":"- High dice scores will comes from organs such as Largeintenstine, Prostate and Spleen due to their higher percentage of the positive label\n- Lower dice scores will comes from organs such as kidney and lungs due to their very sparce masks","metadata":{}},{"cell_type":"code","source":"def create_patches(img_id, rle, width, height, wh=3000, window=1000, save_path='/kaggle/working/train_patches/', min_patch_size=500, testing=False):\n    \n    def check_pad(img, wh=3000, window=500, mask=False):\n        if width == wh:\n            return img\n        if width > wh:\n            return img[:wh, :wh]\n        \n        if mask:\n            pimage = np.zeros((wh, wh), dtype='uint8')\n        else:\n            pimage = np.zeros((wh, wh, 3), dtype='uint8')\n        \n        pimage[:img.shape[1], :img.shape[0]] = img\n        pimage[img.shape[1]:, img.shape[0]:] = img[:(wh-img.shape[1]), :(wh-img.shape[0])]\n        pimage[img.shape[1]:, :img.shape[0]] = img[:(wh-img.shape[1]), :img.shape[0]]\n        pimage[:img.shape[0], img.shape[1]:] = img[:img.shape[0], :(wh-img.shape[1])]\n        return pimage\n            \n    img = tiff.imread(img_path + '{}.tiff'.format(img_id))\n    img = check_pad(img)\n\n    mask = rle_decode(rle, shape=(width, height))\n    mask = check_pad(mask, mask=True)\n    \n    img_patches = []\n    mask_patches = []\n    patches = int((wh/window)**2)\n    possible_negative = np.array(range(patches)).reshape(3,3)[1:-1, 1:-1].flatten().tolist()\n    idx = 0\n    for r in range(0, wh, window):\n        for c in range(0, wh, window):\n            img_patch = img[r: r+window, c: c+window]\n            img_patches.append(img_patch)\n            mask_patch = mask[r: r+window, c: c+window]\n            mask_patches.append(mask_patch)\n            if not testing:\n                if mask_patch.sum() > min_patch_size:\n                    tiff.imwrite(file=save_path + 'image_{}_{}.png'.format(img_id, idx), data=img_patch)\n                    tiff.imwrite(file=save_path + 'mask_{}_{}.png'.format(img_id, idx), data=mask_patch)\n                elif (mask_patch.sum() == 0) and (idx in possible_negative):\n                    tiff.imwrite(file=save_path +  'neg_{}_{}.png'.format(img_id, idx), data=img_patch)\n    \n            idx += 1\n    \n    if testing:\n        return img_patches, mask_patches\n    return True","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-17T13:42:17.380788Z","iopub.execute_input":"2022-07-17T13:42:17.381204Z","iopub.status.idle":"2022-07-17T13:42:17.398259Z","shell.execute_reply.started":"2022-07-17T13:42:17.381169Z","shell.execute_reply":"2022-07-17T13:42:17.397274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/train_patches/\ndf['shape'] = df.apply(lambda x: create_patches(x['id'], x['rle'], x['img_width'], x['img_height']), axis=1)\n#df[df['img_height'].ne(3000)].apply(lambda x: create_patches(x['id'], x['rle'], x['img_width'], x['img_height']), axis=1)\n\nimg_patches, mask_patches = create_patches(name, rle, width, height, testing=True)\nfig, ax = plt.subplots(3, 3, figsize=(16, 10), sharey=True)\nfor idx, i in enumerate(ax.flatten()):   \n    i.imshow(img_patches[idx])\n    i.imshow(mask_patches[idx], cmap='cool', alpha=0.5)\n    i.set_title(idx)\n    i.axis(\"off\")\nplt.tight_layout()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-07-17T13:43:13.152936Z","iopub.execute_input":"2022-07-17T13:43:13.153380Z","iopub.status.idle":"2022-07-17T13:43:25.307835Z","shell.execute_reply.started":"2022-07-17T13:43:13.153345Z","shell.execute_reply":"2022-07-17T13:43:25.306754Z"},"trusted":true},"execution_count":null,"outputs":[]}]}