{"cells":[{"metadata":{},"cell_type":"markdown","source":"![](https://upload.wikimedia.org/wikipedia/commons/d/d6/Blausen_0592_KidneyAnatomy_01.png)\n# **Hacking the kidney**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pylab as pl\nimport seaborn as sns\nimport cv2\nimport json\nimport tifffile\n\n#color\nfrom colorama import Fore, Back, Style","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"BASE_PATH = \"../input/hubmap-kidney-segmentation\"\nTRAIN_PATH = os.path.join(BASE_PATH, \"train\")\n\nprint(os.listdir(BASE_PATH))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Datatrain = pd.read_csv(\n    os.path.join(BASE_PATH, \"train.csv\")\n)\nDatatrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DataSub = pd.read_csv(\n    os.path.join(BASE_PATH, \"sample_submission.csv\"))\nDataSub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Number of train images: {Datatrain.shape[0]}\")\nprint(f\"Number of test images: {DataSub.shape[0]}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DataInfo = pd.read_csv(\n    os.path.join(BASE_PATH, \"HuBMAP-20-dataset_information.csv\")\n)\nDataInfo.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DataInfo.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols=['weight_kilograms']\nDataInfo[cols]=DataInfo[cols].fillna(DataInfo.mode().iloc[0])\n\ncols=['height_centimeters']\nDataInfo[cols]=DataInfo[cols].fillna(DataInfo.mode().iloc[0])\n\ncols=['bmi_kg/m^2']\nDataInfo[cols]=DataInfo[cols].fillna(DataInfo.mode().iloc[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DataInfo.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DataInfo.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Statistics description of each features\nDataInfo.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# features which have dtypes object\nDataInfo.select_dtypes(include = ['object']).head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Features which have dtypes int and float\nDataInfo.select_dtypes(include = ['int64','float64']).head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_feature_columns = list(set(DataInfo.columns) - set(DataInfo._get_numeric_data().columns))\ncategorical_feature_columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numerical_feature_columns = list(DataInfo._get_numeric_data().columns)\nnumerical_feature_columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Laterality\nprint(DataInfo.laterality.value_counts())\nsns.countplot(data=DataInfo, x='laterality', color ='Red')\nplt.ylabel('Count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Laterality Vs Sex\nax = DataInfo.groupby(['laterality', 'sex']).size().plot(kind='bar',color = 'Green')\nax.set_title(\"Total Laterality vs Sex\")\nax.set_xlabel(\"(laterality, sex)\")\nax.set_ylabel(\"Count\")\nfor p in ax.patches:\n    ax.annotate(str(format(int(p.get_height()), ',d')), (p.get_x(), p.get_height()*1.01))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = DataInfo.groupby(['percent_cortex', 'sex']).size().plot(kind='bar',color = 'Yellow')\nax.set_title(\"Percentage Cortex Vs Sex\")\nax.set_xlabel(\"(percent_cortex, sex)\")\nax.set_ylabel(\"Count\")\nfor p in ax.patches:\n    ax.annotate(str(format(int(p.get_height()), ',d')), (p.get_x(), p.get_height()*1.01))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = DataInfo.groupby(['percent_medulla', 'sex']).size().plot(kind='bar',color = 'ORANGE')\nax.set_title(\"Percentage of Medulla Vs Sex\")\nax.set_xlabel(\"(percent_medulla, sex)\")\nax.set_ylabel(\"Count\")\nfor p in ax.patches:\n    ax.annotate(str(format(int(p.get_height()), ',d')), (p.get_x(), p.get_height()*1.01))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pl.rcParams['figure.figsize'] = (14, 10)\npl.rcParams['ytick.labelsize'] = 12\npl.rcParams['xtick.labelsize'] = 11\npl.rcParams['axes.labelsize'] = 23\npl.rcParams['legend.fontsize'] = 20\nsns.set_style('ticks')\nc1, c2, c3, c4 = sns.color_palette(\"Set1\", 4)\nDataInfo.hist()\nsns.despine(offset=10, trim=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots(2, 2, figsize=(15, 15))\ntmp1 = DataInfo.loc[(DataInfo.laterality == 'Right'), :]\na = sns.boxplot(x = 'percent_cortex', y = 'weight_kilograms', data = tmp1,ax=axs[0][0])\na.set_title(\"Right Laterality\")\n\ntmp2 = DataInfo.loc[(DataInfo.laterality == 'Left'), :]\nb = sns.boxplot(x = 'percent_cortex', y = 'weight_kilograms', data = tmp2,ax=axs[0][1])\nb.set_title(\"Left Laterality\")\n\ntmp3 = DataInfo.loc[(DataInfo.laterality == 'Right'), :]\nc = sns.boxplot(x = 'percent_medulla', y = 'weight_kilograms', data = tmp3,ax=axs[1][0])\nc.set_title(\"Right Laterality\")\n\ntmp4 = DataInfo.loc[(DataInfo.laterality == 'Left'), :]\nd = sns.boxplot(x = 'percent_medulla', y = 'weight_kilograms', data = tmp4,ax=axs[1][1])\nd.set_title(\"Left Laterality\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.makedirs('../output')\ninput_dir = '../input/hubmap-kidney-segmentation/train'\noutput_dir = '../output'\nimage_list = ['0486052bb', '2f6ecfcdf', 'aaa6a05cc']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_im(im_name, scale_percent):\n    image_path = os.path.join(input_dir, im_name+'.tiff')\n    im_read = tifffile.imread(image_path)\n    width = int(im_read.shape[1] * scale_percent / 100)\n    height = int(im_read.shape[0] * scale_percent / 100)\n    dim = (width, height)\n    print('File name: {}, original size: {}, resized to: {}'.format(im_name, \n                                                                    (im_read.shape[0], im_read.shape[1]), \n                                                                    (width, height)))\n    resized = cv2.resize(im_read, dim, interpolation=cv2.INTER_AREA)\n    image_path = os.path.join(output_dir, ('r_' + im_name+'.tiff'))\n    tifffile.imwrite(image_path, resized)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for im in image_list:\n    resize_im(im, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle2mask(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return\n    Returns numpy array, 1 - mask, 0 - background\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def resize_mask(im_name, scale_percent):\n    im_read = tifffile.imread(os.path.join(input_dir, im_name+'.tiff'))\n    mask_rle = Datatrain[Datatrain[\"id\"] == im_name][\"encoding\"].values[0]\n    mask = rle2mask(Datatrain[Datatrain[\"id\"] == im_name][\"encoding\"].values[0], (im_read.shape[1], im_read.shape[0]))*255\n    width = int(im_read.shape[1] * scale_percent / 100)\n    height = int(im_read.shape[0] * scale_percent / 100)\n    dim = (width, height)\n    print('File name: {}, original size: {}, resized to: {}'.format(im_name, \n                                                                (im_read.shape[0], im_read.shape[1]), \n                                                                (width, height)))\n    resized = cv2.resize(mask, dim, interpolation=cv2.INTER_AREA)\n    image_path = os.path.join(output_dir, ('r_' + im_name+'_m.tiff'))\n    tifffile.imwrite(image_path, resized)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for im in image_list:\n    resize_mask(im, 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir(output_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_image(image_id):\n    fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(16, 32))\n    image_path = os.path.join(output_dir, 'r_{}.tiff'.format(image_id))\n    mask_path = os.path.join(output_dir, 'r_{}_m.tiff'.format(image_id))\n    \n    image = tifffile.imread(image_path)\n    mask = tifffile.imread(mask_path)\n    if len(mask.shape)==2:    \n        hybr = image[:, :, 0] + mask[:, :]/2\n    else:\n        hybr = image[:, :, 0] + mask[:,: , 0]/2\n    ax[0].imshow(image)\n    ax[0].axis('off')\n    ax[0].set_title('Real Image')\n    ax[1].imshow(hybr)\n    ax[1].axis('off')\n    ax[1].set_title('Masks')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nshow_image(image_list[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nshow_image(image_list[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nshow_image(image_list[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle2mask(mask_rle, shape):\n    # https://www.kaggle.com/paulorzp/rle-functions-run-lenght-encode-decode\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T\n\ndef mask_to_contours(image, mask_layer, thickness):\n    \"\"\" converts a mask to contours using OpenCV and draws them on the image\"\"\"\n    contours, hierarchy = cv2.findContours(mask_layer.astype('uint8'), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n    image = cv2.drawContours(image, contours, -1, color=(255,0,0),thickness=thickness)\n    return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Data paths\nimg_path = \"../input/hubmap-kidney-segmentation/train/0486052bb.tiff\"\ntrain_path = \"../input/hubmap-kidney-segmentation/train.csv\"\nimg_id = img_path.split(\"/\")[-1].split(\".\")[0]\nstruct_path = \"../input/hubmap-kidney-segmentation/train/\" + img_id + \"-anatomical-structure.json\"\nglm_path = \"../input/hubmap-kidney-segmentation/train/\" + img_id + \".json\"\n\n#Read the image an generate the binary mask from the RL encoded data\norg_img = cv2.imread(img_path)\nimg = np.copy(org_img)\ndf_train = pd.read_csv(train_path)\nmask = rle2mask(\n    df_train[df_train[\"id\"] == img_id][\"encoding\"].values[0], \n    (img.shape[1], img.shape[0])\n)\n\n#Merge image with glomeruli mask \nimg = mask_to_contours(img, mask, thickness=-1)\n\n#Extract contours of anathomical structures from json file\nfile = open(struct_path)\nstructures_json = json.load(file)\n\n#Merge the image with the contours of the anathomical structures \nfor structure in structures_json:\n    my_class = structure['properties']['classification']['name']\n    if my_class == 'Medulla': #Note that Medulla will be delimited by a green contour\n        A = np.asarray(structure['geometry']['coordinates'])\n        A = [np.swapaxes(A,0,1)]\n        img = cv2.drawContours(img, A, -1, color=(0,255,0),thickness=50)\n    else: #Note that Cortex will be delimited by a blue contour\n        A = np.asarray(structure['geometry']['coordinates'])\n        A = [np.swapaxes(A,0,1)]\n        img = cv2.drawContours(img, A, -1, color=(0,0,255),thickness=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.gcf()\nfig.set_size_inches(7,6)\nplt.imshow(img)\nplt.grid(False)\nprint(\"Notice that the\",Fore.GREEN, \"Medulla\",Style.RESET_ALL ,\"is delimited by the\", Fore.GREEN,\"green\",Style.RESET_ALL,\"line, and the\" ,Fore.BLUE, \"Cortex\",Style.RESET_ALL , \"is delimited by a\", Fore.BLUE,\"blue\",Style.RESET_ALL,\"line\")\nprint(\"The\",Fore.RED, \"red\",Style.RESET_ALL ,\"dots correspond to the\",Fore.RED, \"glomerulli\",Style.RESET_ALL )\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/hubmap-kidney-segmentation/sample_submission.csv')\ndata_to_submit = pd.DataFrame(train)\ndata_to_submit.to_csv('sample_submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}