{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import collections\nimport json\nimport os\nimport uuid\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image, ImageDraw, ImageFilter\nimport tifffile as tiff \nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/hubmap-kidney-segmentation/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/hubmap-kidney-segmentation/train.csv\")\ntrain.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata = pd.read_csv(\"../input/hubmap-kidney-segmentation/HuBMAP-20-dataset_information.csv\")\nmetadata.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samplesubmission = pd.read_csv(\"../input/hubmap-kidney-segmentation/sample_submission.csv\")\nsamplesubmission.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_id_1 = \"aaa6a05cc\"\nimage_1 = tiff.imread('../input/hubmap-kidney-segmentation/train/' + img_id_1 + \".tiff\")\nprint(\"This image's id:\", img_id_1)\nimage_1.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 15))\nplt.imshow(image_1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,8))\nplt.imshow(image_1[5200:5600, 5600:6000, :])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_id_4 = \"e79de561c\"\nimage_4 = tiff.imread('../input/hubmap-kidney-segmentation/train/' + img_id_4 + \".tiff\")\nprint(\"This image's id:\", img_id_4)\nimage_4.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_4 = image_4[0][0].transpose(1, 2, 0)\nimage_4.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nplt.imshow(image_4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask_rle = train[train[\"id\"]==img_id_1][\"encoding\"].iloc[-1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s = mask_rle.split()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shape=((image_1.shape[1], image_1.shape[0]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"starts -= 1\nends = starts + lengths\nimg = np.zeros(shape[0]*shape[1], dtype=np.uint8)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for lo, hi in zip(starts, ends):\n    img[lo:hi] = 1\n    image_reshaped = img.reshape(shape).T","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_reshaped.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,10))\nplt.imshow(image_reshaped, cmap='coolwarm', alpha=0.5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,10))\nplt.imshow(image_1)\nplt.imshow(image_reshaped, cmap='coolwarm', alpha=0.5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/hubmap-kidney-segmentation/train/e79de561c.json\") as f:\n    e79de561c_json = json.load(f)\n    \nprint(\"lenght of json:\", len(e79de561c_json))\nprint(e79de561c_json[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def flatten(l):\n    for el in l:\n        if isinstance(el, collections.abc.Iterable) and not isinstance(el, (str, bytes)):\n            yield from flatten(el)\n        else:\n            yield el\n\ndef draw_structure(structures, im):\n    \"\"\"\n    anatomical_structure: list of points of anatomical_structure poligon.\n    im: numpy array of image read from tiff file.\n    \"\"\"\n    \n    im = Image.fromarray(im)\n    draw = ImageDraw.Draw(im)\n    for structure in structures:\n        structure_flatten = list(flatten(structure[\"geometry\"][\"coordinates\"][0]))\n        structure = []\n        for i in range(0, len(structure_flatten), 2):\n            structure.append(tuple(structure_flatten[i:i+2]))\n        \n        draw.line(structure, width=100, fill='Red')\n    return im","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,8))\nimage_4_with_line = draw_structure(e79de561c_json, image_4)\nplt.imshow(image_4_with_line)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(f\"../input/hubmap-kidney-segmentation/train/{img_id_1}-anatomical-structure.json\") as f:\n    anatomical_structure_json = json.load(f)\n    \nanatomical_structure_json","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,8))\nimage_1_with_line = draw_structure(anatomical_structure_json, image_1)\nplt.imshow(image_1_with_line)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_info = metadata","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_or_test(image_file):\n    id, _ = image_file.split(\".\")\n    if id in list(train[\"id\"]):\n        return \"train\"\n    else:\n        return \"test\"\n    \nds_info[\"category\"] = ds_info[\"image_file\"].map(train_or_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.style.use(\"Solarize_Light2\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 5))\ng = sns.countplot(data=ds_info, x=\"patient_number\", hue=\"category\", palette=sns.color_palette(\"Set2\", 8))\ng.set_title(\"Number of images per patient\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(10,5), gridspec_kw=dict(wspace=0.1, hspace=0.6))\nfig.suptitle(\"race and ethnicity\", fontsize=15)\ng = sns.countplot(data=ds_info, x=\"race\", hue=\"category\", palette=sns.color_palette(\"Set2\", 8),ax=axes[0])\ng.set_title(\"distribution of race\", fontsize=12)\ng = sns.countplot(data=ds_info, x=\"ethnicity\", palette=sns.color_palette(\"Set2\", 8), hue=\"category\",ax=axes[1])\ng.set_title(\"distribution of ethnicity\", fontsize=12)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create figure and Axes. And set title.\nfig, axes = plt.subplots(2, 2, figsize=(10,6), gridspec_kw=dict(wspace=0.1, hspace=0.6))\nfig.suptitle(\"Sex and age\", fontsize=15)\n\n#Too check layout, I'll show text on each Axes.\ngs = axes[0, 1].get_gridspec()\naxes[0, 0].remove()\naxes[1, 0].remove()\n#Add gridspec we got\naxbig = fig.add_subplot(gs[:, 0])\n\ng = sns.countplot(data=ds_info, x=\"sex\", hue=\"category\", palette=sns.color_palette(\"Set2\", 8),ax=axbig)\ng.set_title(\"distribution of sex\", fontsize=12)\n\n#Add three plots.\ng = sns.distplot(ds_info[ds_info[\"category\"]==\"train\"][\"age\"], color=\"tomato\", kde=False, rug=False,ax=axes[0,1])\ng.set(xlim=(30,80))\ng.set(ylim=(0,3))\ng.set_title(\"distribution of age for train\", fontsize=12)\n\ng = sns.distplot(ds_info[ds_info[\"category\"]==\"test\"][\"age\"], color=\"teal\", kde=False, rug=False, ax=axes[1,1])\ng.set(xlim=(30,80))\ng.set(ylim=(0,3))\ng.set_title(\"distribution of age for test\", fontsize=12)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(10,10), gridspec_kw=dict(wspace=0.1, hspace=0.4))\nfig.suptitle(\"Physical information for train\", fontsize=15)\n\n\ng = sns.distplot(ds_info[ds_info[\"category\"]==\"train\"][\"weight_kilograms\"], color=\"tomato\", kde=False, rug=False, ax=axes[0,0])\ng.set(xlim=(55,135))\ng.set(ylim=(0,5))\ng.set_title(\"weight_kilograms\", fontsize=12)\n\ng = sns.distplot(ds_info[ds_info[\"category\"]==\"train\"][\"height_centimeters\"], color=\"tomato\", kde=False, rug=False, ax=axes[0,1])\ng.set(xlim=(155,195))\ng.set(ylim=(0,5))\ng.set_title(\"height_centimeters\", fontsize=12)\n\ng = sns.distplot(ds_info[ds_info[\"category\"]==\"train\"][\"bmi_kg/m^2\"], color=\"tomato\", kde=False, rug=False, ax=axes[1,0])\ng.set(xlim=(22,37.5))\ng.set(ylim=(0,5))\ng.set_title(\"bmi_kg/m^2\", fontsize=12)\n\ng = sns.countplot(ds_info[ds_info[\"category\"]==\"train\"][\"laterality\"], ax=axes[1,1])\ng.set_title(\"laterality\", fontsize=12)\n\n\nfig, axes = plt.subplots(2, 2, figsize=(10,10), gridspec_kw=dict(wspace=0.1, hspace=0.4))\nfig.suptitle(\"Physical information for test\", fontsize=15)\n\n\ng = sns.distplot(ds_info[ds_info[\"category\"]==\"test\"][\"weight_kilograms\"], color=\"teal\", kde=False, rug=False, ax=axes[0,0])\ng.set(xlim=(55,135))\ng.set(ylim=(0,5))\ng.set_title(\"weight_kilograms\", fontsize=12)\n\ng = sns.distplot(ds_info[ds_info[\"category\"]==\"test\"][\"height_centimeters\"], color=\"teal\", kde=False, rug=False, ax=axes[0,1])\ng.set(xlim=(155,195))\ng.set(ylim=(0,5))\ng.set_title(\"height_centimeters\", fontsize=12)\n\ng = sns.distplot(ds_info[ds_info[\"category\"]==\"test\"][\"bmi_kg/m^2\"], color=\"teal\", kde=False, rug=False, ax=axes[1,0])\ng.set(xlim=(22,37.5))\ng.set(ylim=(0,5))\ng.set_title(\"bmi_kg/m^2\", fontsize=12)\n\ng = sns.countplot(ds_info[ds_info[\"category\"]==\"test\"][\"laterality\"], ax=axes[1,1])\ng.set_title(\"laterality\", fontsize=12)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds_info[\"Ratio_of_medulla_to_cortex\"] = ds_info[\"percent_medulla\"] / ds_info[\"percent_cortex\"] ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(10,5), gridspec_kw=dict(wspace=0.1, hspace=0.6))\nfig.suptitle(\"distribution of ratio of medulla to cortex\", fontsize=15)\ng = sns.distplot(ds_info[ds_info[\"category\"]==\"train\"][\"Ratio_of_medulla_to_cortex\"], color=\"tomato\",kde=False, rug=False, ax=axes[0])\ng.set(ylim=(0,5))\ng.set_title(\"train\", fontsize=12)\ng = sns.distplot(ds_info[ds_info[\"category\"]==\"test\"][\"Ratio_of_medulla_to_cortex\"], color=\"teal\", kde=False, rug=False, ax=axes[1])\ng.set(ylim=(0,5))\ng.set_title(\"test\", fontsize=12)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = ds_info.to_csv(\"ds_info.csv\")\nds_info.to_csv(\"submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}