{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sn\nimport cv2\nimport tifffile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BASE_PATH = \"../input/hubmap-kidney-segmentation/\"\nTRAIN_PATH = os.path.join(BASE_PATH, \"train\")\n\nprint(os.listdir(BASE_PATH))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\n    os.path.join(BASE_PATH, \"train.csv\")\n)\ndf_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub = pd.read_csv(\n    os.path.join(BASE_PATH, \"sample_submission.csv\"))\ndf_sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Number of train images: {df_train.shape[0]}\")\nprint(f\"Number of test images: {df_sub.shape[0]}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_info = pd.read_csv(\n    os.path.join(BASE_PATH, \"HuBMAP-20-dataset_information.csv\")\n)\ndf_info.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q -U pip\n!pip install -q -U seaborn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle2mask(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [\n        np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])\n    ]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo : hi] = 1\n    return img.reshape(shape).T\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_image(image_id, scale=None, verbose=1):\n    image = tifffile.imread(\n        os.path.join(BASE_PATH, f\"train/{image_id}.tiff\")\n    )\n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n    \n    mask = rle2mask(\n        df_train[df_train[\"id\"] == image_id][\"encoding\"].values[0], \n        (image.shape[1], image.shape[0])\n    )\n    \n    if verbose:\n        print(f\"[{image_id}] Image shape: {image.shape}\")\n        print(f\"[{image_id}] Mask shape: {mask.shape}\")\n    \n    if scale:\n        new_size = (image.shape[1] // scale, image.shape[0] // scale)\n        image = cv2.resize(image, new_size)\n        mask = cv2.resize(mask, new_size)\n        \n        if verbose:\n            print(f\"[{image_id}] Resized Image shape: {image.shape}\")\n            print(f\"[{image_id}] Resized Mask shape: {mask.shape}\")\n        \n    return image, mask\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_test_image(image_id, scale=None, verbose=1):\n    image = tifffile.imread(\n        os.path.join(BASE_PATH, f\"test/{image_id}.tiff\")\n    )\n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n    \n    if verbose:\n        print(f\"[{image_id}] Image shape: {image.shape}\")\n    \n    if scale:\n        new_size = (image.shape[1] // scale, image.shape[0] // scale)\n        image = cv2.resize(image, new_size)\n        \n        if verbose:\n            print(f\"[{image_id}] Resized Image shape: {image.shape}\")\n        \n    return image\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_image_and_mask(image, mask, image_id):\n    plt.figure(figsize=(16, 10))\n    \n    plt.subplot(1, 3, 1)\n    plt.imshow(image)\n    plt.title(f\"Image {image_id}\", fontsize=18)\n    \n    plt.subplot(1, 3, 2)\n    plt.imshow(image)\n    plt.imshow(mask, cmap=\"hot\", alpha=0.5)\n    plt.title(f\"Image {image_id} + mask\", fontsize=18)    \n    \n    plt.subplot(1, 3, 3)\n    plt.imshow(mask, cmap=\"hot\")\n    plt.title(f\"Mask\", fontsize=18)    \n    \n    plt.show()\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_grid_image_with_mask(image, mask):\n    plt.figure(figsize=(16, 16))\n    \n    w_len = image.shape[0]\n    h_len = image.shape[1]\n    \n    min_len = min(w_len, h_len)\n    w_start = (w_len - min_len) // 2\n    h_start = (h_len - min_len) // 2\n    \n    plt.imshow(image[w_start : w_start + min_len, h_start : h_start + min_len])\n    plt.imshow(\n        mask[w_start : w_start + min_len, h_start : h_start + min_len], cmap=\"hot\", alpha=0.5,\n    )\n    plt.axis(\"off\")\n            \n    plt.show()\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef plot_slice_image_and_mask(image, mask, start_h, end_h, start_w, end_w):\n    plt.figure(figsize=(16, 5))\n    \n    sub_image = image[start_h:end_h, start_w:end_w, :]\n    sub_mask = mask[start_h:end_h, start_w:end_w]\n    \n    plt.subplot(1, 3, 1)\n    plt.imshow(sub_image)\n    plt.axis(\"off\")\n    \n    plt.subplot(1, 3, 2)\n    plt.imshow(sub_image)\n    plt.imshow(sub_mask, cmap=\"hot\", alpha=0.5)\n    plt.axis(\"off\")\n    \n    plt.subplot(1, 3, 3)\n    plt.imshow(sub_mask, cmap=\"hot\")\n    plt.axis(\"off\")\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"small_ids = [\n    \"0486052bb\", \"095bf7a1f\", \"1e2425f28\", \"2f6ecfcdf\",\n    \"54f2eec69\", \"aaa6a05cc\", \"cb2d976f4\", \"e79de561c\",\n]\nsmall_images = []\nsmall_masks = []\n\nfor small_id in small_ids:\n    tmp_image, tmp_mask = read_image(small_id, scale=20, verbose=0)\n    small_images.append(tmp_image)\n    small_masks.append(tmp_mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 16))\nfor ind, (tmp_id, tmp_image) in enumerate(zip(small_ids, small_images)):\n    plt.subplot(3, 3, ind + 1)\n    plt.imshow(tmp_image)\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 16))\nfor ind, (tmp_id, tmp_image, tmp_mask) in enumerate(zip(small_ids, small_images, small_masks)):\n    plt.subplot(3, 3, ind + 1)\n    plt.imshow(tmp_image)\n    plt.imshow(tmp_mask, cmap=\"hot\", alpha=0.5)\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"small_ids = [\n    \"26dc41664\", \"afa5e8098\", \"b2dc8411c\", \"b9a3865fc\", \"c68fe75ea\",\n]\nsmall_images = []\n\nfor small_id in small_ids:\n    tmp_image = read_test_image(small_id, scale=20, verbose=0)\n    small_images.append(tmp_image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 11))\nfor ind, (tmp_id, tmp_image) in enumerate(zip(small_ids, small_images)):\n    plt.subplot(2, 3, ind + 1)\n    plt.imshow(tmp_image)\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = \"0486052bb\"\nimage, mask = read_image(image_id, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image_and_mask(image, mask, image_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_slice_image_and_mask(image, mask, 5000, 7500, 2500, 5000)\nplot_slice_image_and_mask(image, mask, 5250, 5720, 3500, 4000)\nplot_slice_image_and_mask(image, mask, 5375, 5575, 3650, 3850)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_grid_image_with_mask(image, mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = \"095bf7a1f\"\nimage, mask = read_image(image_id, scale=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image_and_mask(image, mask, image_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_slice_image_and_mask(image, mask, 7500, 10000, 10000, 12500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_grid_image_with_mask(image, mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = \"1e2425f28\"\nimage, mask = read_image(image_id, scale=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image_and_mask(image, mask, image_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = \"2f6ecfcdf\"\nimage, mask = read_image(image_id, scale=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image_and_mask(image, mask, image_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_slice_image_and_mask(image, mask, 10000, 12000, 8000, 10000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = \"aaa6a05cc\"\nimage, mask = read_image(image_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image_and_mask(image, mask, image_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_slice_image_and_mask(image, mask, 6500, 8500, 7000, 9000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_id = \"e79de561c\"\nimage, mask = read_image(image_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_image_and_mask(image, mask, image_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_slice_image_and_mask(image, mask, 4000, 6000, 2000, 4000)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metadata Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_json(\n    os.path.join(BASE_PATH, \"train/0486052bb-anatomical-structure.json\")\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_json(\n    os.path.join(BASE_PATH, \"train/0486052bb.json\")\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_info[\"split\"] = \"test\"\ndf_info.loc[df_info[\"image_file\"].isin(os.listdir(os.path.join(BASE_PATH, \"train\"))), \"split\"] = \"train\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_info[\"area\"] = df_info[\"width_pixels\"] * df_info[\"height_pixels\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_info.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 35))\nplt.subplot(6, 2, 1)\nsn.countplot(x=\"race\", hue=\"split\", data=df_info)\nplt.subplot(6, 2, 2)\nsn.countplot(x=\"ethnicity\", hue=\"split\", data=df_info)\nplt.subplot(6, 2, 3)\nsn.countplot(x=\"sex\", hue=\"split\", data=df_info)\nplt.subplot(6, 2, 4)\nsn.countplot(x=\"laterality\", hue=\"split\", data=df_info)\nplt.subplot(6, 2, 5)\nsn.histplot(x=\"age\", hue=\"split\", data=df_info)\nplt.subplot(6, 2, 6)\nsn.histplot(x=\"weight_kilograms\", hue=\"split\", data=df_info)\nplt.subplot(6, 2, 7)\nsn.histplot(x=\"height_centimeters\", hue=\"split\", data=df_info)\nplt.subplot(6, 2, 8)\nsn.histplot(x=\"bmi_kg/m^2\", hue=\"split\", data=df_info)\nplt.subplot(6, 2, 9)\nsn.histplot(x=\"percent_cortex\", hue=\"split\", data=df_info)\nplt.subplot(6, 2, 10)\nsn.histplot(x=\"percent_medulla\", hue=\"split\", data=df_info)\nplt.subplot(6, 2, 11)\nsn.histplot(x=\"area\", hue=\"split\", data=df_info);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Building"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle2mask(mask_rle, shape=(1600,256)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    #print(s)\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        \n        img[lo:hi] = 1\n        \n    return img.reshape(shape).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_encode_less_memory(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    This simplified method requires first and last pixel to be zero\n    '''\n    pixels = img.T.flatten()\n    \n    # This simplified method requires first and last pixel to be zero\n    pixels[0] = 0\n    pixels[-1] = 0\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 2\n    runs[1::2] -= runs[::2]\n    \n    return ' '.join(str(x) for x in runs)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_size_dict(img_id, x, y):\n   # print(x,y)\n    image_id = [thing[:-5] for thing in img_id]\n    print(image_id)\n    x_y = [(x[i], y[i]) for i in range(0, len(x))]  \n    print(x_y)\n    t= dict(zip(image_id, x_y))\n    print(t)\n    return t","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def global_shift_mask(maskpred1, y_shift, x_shift):\n    \"\"\"\n    applies a global shift to a mask by padding one side and cropping from the other\n    \"\"\"\n    if y_shift <0 and x_shift >=0:\n        maskpred2 = np.pad(maskpred1, [(0,abs(y_shift)), (abs(x_shift), 0)], mode='constant', constant_values=0)\n        maskpred3 = maskpred2[abs(y_shift):, :maskpred1.shape[1]]\n    elif y_shift >=0 and x_shift <0:\n        maskpred2 = np.pad(maskpred1, [(abs(y_shift),0), (0, abs(x_shift))], mode='constant', constant_values=0)\n        maskpred3 = maskpred2[:maskpred1.shape[0], abs(x_shift):]\n    elif y_shift >=0 and x_shift >=0:\n        maskpred2 = np.pad(maskpred1, [(abs(y_shift),0), (abs(x_shift), 0)], mode='constant', constant_values=0)\n        maskpred3 = maskpred2[:maskpred1.shape[0], :maskpred1.shape[1]]\n    elif y_shift < 0 and x_shift < 0:\n        maskpred2 = np.pad(maskpred1, [(0, abs(y_shift)), (0, abs(x_shift))], mode='constant', constant_values=0)\n        maskpred3 = maskpred2[abs(y_shift):, abs(x_shift):]\n    return maskpred3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfpred = pd.read_csv('../input/submission-public-tpu/submission_public_TPU.csv')\nTARGET_ID = 'afa5e8098'\ny_shift = -48 #-40 -> -42 -> -43\nx_shift = -21 #-24 -> -20 -> -21","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#get image sizes \ndfinfo = pd.read_csv('../input/hubmap-kidney-segmentation/HuBMAP-20-dataset_information.csv')\n\nsize_dict = image_size_dict(dfinfo.image_file, dfinfo.width_pixels, dfinfo.height_pixels)  #dict which contains image sizes mapped to id's\nmask_shape = size_dict.get(TARGET_ID)\n#print(mask_shape)\ntaridx = dfpred[dfpred['id']==TARGET_ID].index.values[0]  #row of TARGET_ID in dfpred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"maskpred = rle2mask(dfpred.iloc[taridx]['predicted'], mask_shape)\n\nmaskpred1 = maskpred.copy()\nmaskpred1[maskpred1>0]=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmask_shifted = global_shift_mask(maskpred1, y_shift, x_shift)  #apply specified shift to mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"newrle = rle_encode_less_memory(mask_shifted)  #rle encode shifted mask\n\ndfpred.at[taridx, 'predicted'] = newrle\n\ndfsample = pd.read_csv('../input/hubmap-kidney-segmentation/sample_submission.csv')\n\nmydict = dict(zip(dfpred['id'], dfpred['predicted']))\n\ndfsample['predicted'] = dfsample['id'].map(mydict).fillna(dfsample['predicted'])\n\ndfsample = dfsample.replace(np.nan, '', regex=True)\n\ndfsample.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfsample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output = pd.read_csv('./submission.csv')\noutput.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}