{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Contents \n1. [Libs & functions](#1)\n2. [Summary](#2)\n    * 2.1 [Source dataframe files](#2.1)\n3. [Datasets preparation](#3)\n    * 3.1 [Example of train dataframe](#3.1)\n    * 3.2 [Example of test dataframe](#3.2)\n4. [Images](#4)\n    * 4.1 [Image file names](#4.1)\n    * 4.2 [Train images visualization](#4.2)\n    * 4.3 [Test images visualization](#4.3)\n5. [Metadata EDA](#5)\n6. [Glomerulus cells EDA](#6)\n    * 6.1 [Correlations](#6.1)\n    * 6.2 [Regplot glomerulus cells on pix / bmi_kg/m^2](#6.2)\n"},{"metadata":{"id":"AGUssHI3IW34"},"cell_type":"markdown","source":"# <a id='1'>Libs & functions</a>"},{"metadata":{"id":"2Msd4DiE4bw5","_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"import PIL\nimport gc\nimport random\nimport tifffile\nimport cv2\nimport json\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nPATH = \"../input/hubmap-kidney-segmentation/\"\nPATH_TRAIN = PATH + \"train/\"\nPATH_TEST = PATH + \"test/\"","execution_count":null,"outputs":[]},{"metadata":{"id":"HanullvVBiFR","_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# usefull functions\n\ndef js2df(path):\n    '''\n    Convert json to DataFrame\n    '''\n    with open(path) as f:\n        data = json.load(f)\n    return pd.DataFrame(data)\n\ndef shoelace_formula(mat):\n    '''\n    Formula for find area of polygon, mat is matrix with coords of size [Ncoords, 2]\n    '''\n    m = mat.squeeze()\n    sum1 = np.sum(m[:-1, 0] * m[1:, 1]) + m[-1, 0] * m[0, 1] \n    sum2 = np.sum(m[1:, 0] * m[:-1, 1]) + m[0, 0] * m[-1, 1]\n    return np.abs(sum1 - sum2) / 2\n\ndef open_img(path):\n    '''\n    Open .tiff image\n    '''\n    image = np.array(tifffile.imread(path))\n    if image.ndim != 3:\n        image = np.squeeze(image).transpose(1, 2, 0)\n    return image\n\ndef rescale_img(image, scale=1):\n    '''\n    Rescale image\n    '''\n    dim = (int(image.shape[1]*scale), int(image.shape[0]*scale))\n    image = cv2.resize(image, dim)\n    return image\n\ndef mask_img(image, mask_list, scale=1, rgbcolor=(70, 255, 0), imgw=0.4, maskw=1):\n    '''\n    Masking image using list of polygons(mask)\n    '''\n    mask_polygons = [(i*scale).astype(int) for i in mask_list]\n    mask = np.zeros(image.shape, dtype=np.uint8)\n    mask = cv2.fillPoly(mask, mask_polygons, color=rgbcolor)\n    return cv2.addWeighted(image, imgw, mask, maskw, 0)\n\ndef zoom_img(img, pos1, pos2, scale=1):\n    '''\n    Return rectangle pf image with coords in pos1(start position), pos2(end position)\n    '''\n    x1, y1 = int(pos1[0]*scale), int(pos1[1]*scale)\n    x2, y2 = int(pos2[0]*scale), int(pos2[1]*scale)\n    return img[y1:y2, x1:x2, :]\n\ndef my_style_ax(ax, title='', xlabel='', ylabel='', tfontsize=15, ylfontsize=18):\n    ax.set_title(title, fontsize=tfontsize)\n    ax.get_yaxis().set_ticks([])\n    ax.set_xlabel(xlabel)\n    ax.set_ylabel(ylabel, rotation=0, labelpad=45, fontsize=ylfontsize)\n    ax.spines['right'].set_visible(False)\n    ax.spines['top'].set_visible(False)\n    ax.spines['left'].set_visible(False)\n\ndef preproccess_traindf(traincsv_path, infocsv_path, train_path):\n    '''\n    Preprocess train dataframe, for train add following data:\n        anatomical_structures_segmention_geometry(dict with coordinates of polygons of cortex/medulla segments)\n        kidney_bbox(kidney bounding box)\n        glomerulus_coords(list of glom. coordinates)\n        glom_num(number of glomerulus cells)\n        avg_g_area(average area of glom. cell in pixels)\n        glor_cells_on_pix(the content of the pixels of the cells of the cell on one pixel of the cortex)\n    '''\n    # get train dfs, train names\n    train_names = [i + '.tiff' for i in list(pd.read_csv(traincsv_path)['id'])]\n    df_info = pd.read_csv(infocsv_path)\n    df_train = df_info[df_info['image_file'].isin(train_names)].reset_index(drop=True)\n    objects_num = len(df_train)\n\n    # new columns: anatomical geometry(cortex and etc.), kidney bbox\n    anat_struct_res = []\n    kidney_bbox_res = []\n    kidney_area_res = []\n    for i in range(objects_num):\n        # anatomical coords\n        df_anatom = js2df(train_path + df_train['anatomical_structures_segmention_file'][i])\n        geometry = dict()\n        for j in range(len(df_anatom)):\n            name = df_anatom['properties'][j]['classification']['name']\n            coords = np.array(df_anatom['geometry'][j]['coordinates'])\n            if coords.ndim == 2:\n                nnum = 0\n                for nnum in range(len(coords)):\n                    geometry[name + str(nnum)] = np.array([coords[nnum][0]])\n            else:         \n                geometry[name] = coords\n        anat_struct_res.append(geometry)\n        # get kidney bbox\n        acords = np.hstack(list(geometry.values())).squeeze()\n        kidney_bbox_res.append([np.min(acords[:, 0]), np.min(acords[:, 1]), np.max(acords[:, 0]), np.max(acords[:, 1])])\n        # kidney area\n        kidney_area = 0\n        for k, v in geometry.items():\n            if 'cortex' in str(k).lower():\n                kidney_area += shoelace_formula(v)\n        kidney_area_res.append(kidney_area)\n    # add in dataframe\n    df_train['anatomical_structures_segmention_geometry'] = anat_struct_res\n    df_train['kidney_bbox'] = kidney_bbox_res\n    df_train['kidney_area'] = kidney_area_res\n\n    # new columns: glomerulus_coords, glom_num, avg_g_area\n    glom_coords_res = []\n    glom_num_res = []\n    avg_g_area_res = []\n    for i in range(objects_num):\n        # glomerulus cells coordinates                                                                          \n        df_glom = js2df(train_path + df_train['glomerulus_segmentation_file'][i])\n        glomerulus_coords = []\n        g_area = 0\n        for k in range(len(df_glom)):\n            g_coords = np.array(df_glom['geometry'][k]['coordinates'])\n            g_area += shoelace_formula(g_coords)\n            glomerulus_coords.append(g_coords)\n        glom_coords_res.append(glomerulus_coords)\n        # number of glomeruluses\n        glom_num_res.append(len(df_glom))\n        # average size of glomeruluses cells\n        avg_g_area_res.append(g_area / len(df_glom))\n    # add in dataframe\n    df_train['glomerulus_coords'] = glom_coords_res\n    df_train['glom_num'] = glom_num_res\n    df_train['avg_g_area'] = avg_g_area_res\n    df_train['glor_cells_on_pix'] = df_train['glom_num']*df_train['avg_g_area'] / df_train['kidney_area'] \n\n    del df_info, train_names, anat_struct_res, kidney_bbox_res, glom_coords_res, glom_num_res, \n    avg_g_area_res, kidney_area_res\n    gc.collect()\n    return df_train  \n\n\ndef preproccess_testdf(traincsv_path, infocsv_path, test_path):\n    '''\n    Preprocess train dataframe, for train add following data:\n        anatomical_structures_segmention_geometry(dict with coordinates of polygons of cortex/medulla)\n        kidney_bbox(kidney bounding box)\n    '''\n    # get test dfs, train names\n    train_names = [i + '.tiff' for i in list(pd.read_csv(traincsv_path)['id'])]\n    df_info = pd.read_csv(infocsv_path)\n    df_test = df_info[~df_info['image_file'].isin(train_names)].reset_index(drop=True)\n    objects_num = len(df_test)\n\n    # new columns: anatomical geometry(cortex and etc.), kidney bbox\n    anat_struct_res = []\n    kidney_bbox_res = []\n    for i in range(objects_num):\n        # anatomical coords\n        df_anatom = js2df(test_path + df_test['anatomical_structures_segmention_file'][i])\n        geometry = dict()\n        for j in range(len(df_anatom)):\n            name = df_anatom['properties'][j]['classification']['name']\n            coords = np.array(df_anatom['geometry'][j]['coordinates'])\n            if coords.ndim == 2:\n                nnum = 0\n                for nnum in range(len(coords)):\n                    geometry[name + str(nnum)] = np.array([coords[nnum][0]])\n            else:         \n                geometry[name] = coords\n        anat_struct_res.append(geometry)\n        # get kidney bbox\n        acords = np.hstack(list(geometry.values())).squeeze()\n        kidney_bbox_res.append([np.min(acords[:, 0]), np.min(acords[:, 1]), np.max(acords[:, 0]), np.max(acords[:, 1])])\n    # add in dataframe\n    df_test['anatomical_structures_segmention_geometry'] = anat_struct_res\n    df_test['kidney_bbox'] = kidney_bbox_res \n\n    del df_info, train_names, anat_struct_res, kidney_bbox_res\n    gc.collect()\n    return df_test","execution_count":null,"outputs":[]},{"metadata":{"id":"I4BkkhIpE0XH"},"cell_type":"markdown","source":"# <a id='2'>Summary</a>"},{"metadata":{"id":"hopmI2ECFDDN","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# get original datasets\ndf_train = pd.read_csv(PATH + 'train.csv')\ndf_info = pd.read_csv(PATH + 'HuBMAP-20-dataset_information.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- # <a id='2.1'>Source dataframe files</a>"},{"metadata":{},"cell_type":"markdown","source":"\n**train.csv**\n\n- *id* - id of image\n- *encoding* - RLE encoded segmentation masks"},{"metadata":{"id":"zBH8KLsWF0ag","outputId":"cd9d6cae-0f48-415e-fefd-b39425ef56fe","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(\"-----------------train.csv-----------------\")\nprint(\"length: \", len(df_train))\ndf_train_info = pd.DataFrame()\ndf_train_info['type'] = df_train.dtypes\ndf_train_info['nans'] = df_train.isnull().sum()\ndf_train_info['not-null count'] = df_train.notna().sum()\ndf_train_info['unique values'] = df_train.apply(lambda col: col.nunique())\ndf_train_info\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"df_train.sample(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**HuBMAP-20-dataset_information.csv**\n\n- *image_file* - name of image file in .tiff format\n- *width_pixels* - image width\n- *height_pixels* - image height\n- *anatomical_structures_segmention_file* - name of .json file, storing segments(polygons) of kidney parts(cortex/medulla)\n- *glomerulus_segmentation_file* - name of .json file, storing segments(polygons) of glomerulus cells\n- *patient_number* - patient number\n- *race* - race of patient\n- *sex* - patient gender\n- *ethnicity* ethnicity of patient\n- *age* - patient age\n- *weight_kilograms* - weight of patient in kg. \n- *height_centimeters* - height of patient in cm.\n- *bmi_kg/m^2* - body mass index(weight_kilograms / height_centimeters^2)\n- *laterality* - laterality of kidney(left / right)\n- *percent_cortex* percent of cortex(outer part of the kidney)\n- *percent_medulla* percent of medulla(inner part of the kidney)"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(\"-----------------HuBMAP-20-dataset_information.csv-----------------\")\nprint(\"length: \", len(df_info))\ndf_info_info = pd.DataFrame()\ndf_info_info['type'] = df_info.dtypes\ndf_info_info['nans'] = df_info.isnull().sum()\ndf_info_info['not-null count'] = df_info.notna().sum()\ndf_info_info['unique values'] = df_info.apply(lambda col: col.nunique())\ndf_info_info","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"df_info.sample(2)","execution_count":null,"outputs":[]},{"metadata":{"id":"s_fd6ZnZHjoY"},"cell_type":"markdown","source":"# <a id='3'>Datasets preparation</a>\nHere i formed two dataFrames for simplify work with dataset: \n- trainc.csv, added columns:\n    - **anatomical_structures_segmention_geometry**, dict with coordinates of polygons of cortex/medulla segments\n    - **kidney_bbox**, kidney bounding box\n    - **glomerulus_coords**, list of glomerulus polygons coordinates\n    - **glom_num**, number of glomerulus cells\n    - **avg_g_area**, average area of glom. cell in pixels\n    - **glor_cells_on_pix**, the content of the pixels of the glomerulus cells of the cell on one pixel of the cortex\n- test.csv, added columns:\n    - **anatomical_structures_segmention_geometry**\n    - **kidney_bbox**"},{"metadata":{"id":"TPfR9Db6vW_W","_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# get preprocessed datasets\ndf_train = preproccess_traindf(PATH+'train.csv', PATH + 'HuBMAP-20-dataset_information.csv', PATH_TRAIN)\ndf_test = preproccess_testdf(PATH+'train.csv', PATH + 'HuBMAP-20-dataset_information.csv', PATH_TEST)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<a id='3.1'>Example of train dataframe</a>"},{"metadata":{"id":"XEqNE7pDFvOT","outputId":"de44517e-9478-4ed6-ccae-f8776e115300","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# example of train dataframe\ndf_train[:2]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<a id='3.2'>Example of test dataframe</a>"},{"metadata":{"id":"IIDPdvNMMBhf","outputId":"c81fb01d-5d83-41c0-da12-ce8ce758d877","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# example of test dataframe\ndf_test[:2]","execution_count":null,"outputs":[]},{"metadata":{"id":"fuwOgF98MQMP"},"cell_type":"markdown","source":"# <a id='4'>Images</a>"},{"metadata":{},"cell_type":"markdown","source":"- # <a id='4.1'>Image file names</a>"},{"metadata":{"id":"r2ZYrnAXL6NY","outputId":"7fd09191-0b02-4ecd-b798-dc4e000fecab","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"print(\"Train images: \", df_train['image_file'].to_list())\nprint(\"Test images: \", df_test['image_file'].to_list())","execution_count":null,"outputs":[]},{"metadata":{"id":"Qxa1OeKrMeR_"},"cell_type":"markdown","source":"- # <a id='4.2'>Train images visualization</a>"},{"metadata":{"id":"5rPtbSpUG3bC","outputId":"38810cfc-6640-45b1-bee6-1fa0a426c1ab","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# draw train samples\nSCALE_COEF = 0.1\nfig, ax = plt.subplots(ncols=3, nrows=8, figsize=(16, 50))\nfig.subplots_adjust(wspace=0, hspace=0)\n\n\nfor i in range(8):\n    img_name = df_train['image_file'][i]\n    img_width, img_height = df_train['width_pixels'][i], df_train['height_pixels'][i]\n    img_gl = df_train['glomerulus_coords'][i]\n    img_bbox = [int(i*SCALE_COEF) for i in df_train['kidney_bbox'][i]]\n    img_anatomy = df_train['anatomical_structures_segmention_geometry'][i]\n    img_anatomy = {i:(img_anatomy[i]*SCALE_COEF).astype('int32') for i in img_anatomy.keys()}\n\n    # original image\n    img = open_img(PATH_TRAIN + img_name)\n    img = rescale_img(img, SCALE_COEF)\n    ax[i][0].imshow(img)\n    ax[i][0].set_title(f\"Orig. {img_name}, {img_width}x{img_height}\", fontsize=13)\n    ax[i][0].axis('off')\n\n\n    # bboxed image, add anatomical structures segmentation polygons\n    anatomical_segmented_img = img.copy()\n    # draw bbox for kidney\n    anatomical_segmented_img = cv2.rectangle(anatomical_segmented_img, \n                                             (img_bbox[0], img_bbox[1]), (img_bbox[2], img_bbox[3]), \n                                             (0, 0, 255), 20)\n    # create segments mask\n    for k, v in img_anatomy.items():\n        mask_polygons = v.squeeze()\n        mask = np.zeros(img.shape, dtype=np.uint8)\n        rand_colors = tuple(random.randint(0, 255) for i in range(3))\n        mask = cv2.fillConvexPoly(mask, mask_polygons, color=rand_colors)\n        anatomical_segmented_img = cv2.addWeighted(anatomical_segmented_img, 1, mask, 1, 0)\n\n        # put text on segments\n        M = cv2.moments(mask_polygons)\n        if M[\"m10\"] == 0 or  M[\"m00\"] == 0 or M[\"m01\"] == 0 or M[\"m00\"] == 0:\n            continue\n        else:\n            x, y = int(M[\"m10\"] / M[\"m00\"]), int(M[\"m01\"] / M[\"m00\"])\n            anatomical_segmented_img = cv2.putText(anatomical_segmented_img, k, (x-200, y), \n                                    cv2.FONT_HERSHEY_SIMPLEX, \n                                    5, \n                                    (0,0,0), \n                                    10, cv2.LINE_AA)\n    ax[i][1].imshow(anatomical_segmented_img)\n    ax[i][1].set_title(f\"Anatomicaly segmented {img_name}\", fontsize=13)\n    ax[i][1].axis('off')\n\n    # mask\n    masked_img = img.copy()\n    masked_img = mask_img(masked_img, img_gl, scale=SCALE_COEF)\n    ax[i][2].imshow(masked_img)\n    ax[i][2].set_title(f\"Masked {img_name}\", fontsize=13)\n    ax[i][2].axis('off')\n","execution_count":null,"outputs":[]},{"metadata":{"id":"z04JpdlFsVNY","outputId":"abecf82d-e78d-4b7a-abd0-d893d80e3c36","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"SCALE_COEF = 1\nfig, ax = plt.subplots(ncols=2, nrows=2, figsize=(10, 7))\nfig.subplots_adjust(hspace=0)\nax = ax.flatten()\n\n\nimg_filename =  df_train['image_file'][2]\nimg_gl = df_train['glomerulus_coords'][2]\nimg_bbox_kidney = df_train['kidney_bbox'][2]\n\nimg = open_img(PATH_TRAIN + img_filename)\nimg = rescale_img(img, SCALE_COEF)\nimg = mask_img(img, img_gl, maskw=0.4, scale=SCALE_COEF)\n\nzoomed1 = zoom_img(img, (img_bbox_kidney[0], img_bbox_kidney[1]), (img_bbox_kidney[2], img_bbox_kidney[3]), scale=SCALE_COEF)\nax[0].imshow(zoomed1)\nzoomed2 = zoom_img(zoomed1, (0, 2500), (20000, 22500))\nax[1].imshow(zoomed2)\nzoomed3 = zoom_img(zoomed2, (5000, 4000), (15000, 8000))\nax[2].imshow(zoomed3)\nzoomed4 = zoom_img(zoomed3, (2200, 2250), (2456, 2506))\nax[3].imshow(zoomed4)","execution_count":null,"outputs":[]},{"metadata":{"id":"cvRS7BXlMrd3"},"cell_type":"markdown","source":"<a id='4.3'>Test images visualization</a>"},{"metadata":{"id":"4d3booGaF_ww","outputId":"d5c00d3d-09a0-4de6-a741-1a1526b713ee","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# draw test samples\nSCALE_COEF = 0.1\nfig, ax = plt.subplots(ncols=2, nrows=5, figsize=(16, 30))\nfig.subplots_adjust(wspace=0, hspace=0)\n\n\nfor i in range(5):\n    img_name = df_test['image_file'][i]\n    img_width, img_height = df_test['width_pixels'][i], df_test['height_pixels'][i]\n    img_bbox = [int(i*SCALE_COEF) for i in df_test['kidney_bbox'][i]]\n    img_anatomy = df_test['anatomical_structures_segmention_geometry'][i]\n    img_anatomy = {i:(img_anatomy[i]*SCALE_COEF).astype('int32') for i in img_anatomy.keys()}\n\n    # original image\n    img = open_img(PATH_TEST + img_name)\n    img = rescale_img(img, SCALE_COEF)\n    ax[i][0].imshow(img)\n    ax[i][0].set_title(f\"Orig. {img_name}, {img_width}x{img_height}\", fontsize=13)\n    ax[i][0].axis('off')\n\n\n    # bboxed image, add anatomical structures segmentation polygons\n    anatomical_segmented_img = img.copy()\n    # draw bbox for kidney\n    anatomical_segmented_img = cv2.rectangle(anatomical_segmented_img, \n                                             (img_bbox[0], img_bbox[1]), (img_bbox[2], img_bbox[3]), \n                                             (0, 0, 255), 20)\n    # create segments mask\n    for k, v in img_anatomy.items():\n        mask_polygons = v.squeeze()\n        mask = np.zeros(img.shape, dtype=np.uint8)\n        rand_colors = tuple(random.randint(0, 255) for i in range(3))\n        mask = cv2.fillConvexPoly(mask, mask_polygons, color=rand_colors)\n        anatomical_segmented_img = cv2.addWeighted(anatomical_segmented_img, 1, mask, 1, 0)\n\n        # put text on segments\n        M = cv2.moments(mask_polygons)\n        if M[\"m10\"] == 0 or  M[\"m00\"] == 0 or M[\"m01\"] == 0 or M[\"m00\"] == 0:\n            continue\n        else:\n            x, y = int(M[\"m10\"] / M[\"m00\"]), int(M[\"m01\"] / M[\"m00\"])\n            anatomical_segmented_img = cv2.putText(anatomical_segmented_img, k, (x-200, y), \n                                    cv2.FONT_HERSHEY_SIMPLEX, \n                                    5, \n                                    (0,0,0), \n                                    10, cv2.LINE_AA)\n    ax[i][1].imshow(anatomical_segmented_img)\n    ax[i][1].set_title(f\"Anatomicaly segmented {img_name}\", fontsize=13)\n    ax[i][1].axis('off')","execution_count":null,"outputs":[]},{"metadata":{"id":"H1MQ2CaJMuqH"},"cell_type":"markdown","source":"# <a id='5'>Metadata EDA</a>\n- Area of image\n- Sex\n- Age\n- Body mass index\n- Laterality\n- Race"},{"metadata":{"id":"BLFsazjL-Ts-","outputId":"fd30bba3-0284-410a-87f8-0e6a281bc2fa","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# metadata EDA\n# average area of kidney\nsns.set_style(\"white\")\nfig, ax = plt.subplots(ncols=2, nrows=6, figsize=(12, 40))\nfig.subplots_adjust(wspace=0)\n\n# mean image area\ntrain_areas = df_train['width_pixels'] * df_train['height_pixels'] / 10**5\ntrain_area_min = str(df_train['width_pixels'][np.argmin(train_areas)]) + 'x' + str(df_train['height_pixels'][np.argmin(train_areas)])\ntrain_area_max = str(df_train['width_pixels'][np.argmax(train_areas)]) + 'x' + str(df_train['height_pixels'][np.argmax(train_areas)])\nsns.distplot(train_areas, ax=ax[0][0])\nmy_style_ax(\n    ax[0][0],\n    title=f\"TRAIN | min: {train_area_min}, max: {train_area_max}\",\n    xlabel=\"pix*10^5\",\n    ylabel='Images area',\n    tfontsize=13\n)\ntest_areas = df_test['width_pixels'] * df_test['height_pixels'] / 10**5 \ntest_area_min = str(df_test['width_pixels'][np.argmin(test_areas)]) + 'x' + str(df_test['height_pixels'][np.argmin(test_areas)])\ntest_area_max = str(df_test['width_pixels'][np.argmax(test_areas)]) + 'x' + str(df_test['height_pixels'][np.argmax(test_areas)])\nsns.distplot(test_areas, ax=ax[0][1])\nmy_style_ax(\n    ax[0][1],\n    title=f\"TEST | min: {test_area_min}, max: {test_area_max}\",\n    xlabel=\"pix*10^5\",\n    tfontsize=13\n)\n\n# sex\ntrain_sex = df_train['sex']\nsns.countplot(train_sex, ax=ax[1][0])\nmy_style_ax(\n    ax[1][0],\n    title=f\"TRAIN | {train_sex.value_counts()['Male']} male / {train_sex.value_counts()['Female']} female\",\n    ylabel='Sex'\n)\ntest_sex = df_test['sex']\nsns.countplot(test_sex, ax=ax[1][1])\nmy_style_ax(\n    ax[1][1],\n    title=f\"TEST | {test_sex.value_counts()['Male']} male / {test_sex.value_counts()['Female']} female\"\n)\n\n# age\ntrain_age = df_train['age']\nsns.distplot(train_age, ax=ax[2][0])\nmy_style_ax(\n    ax[2][0],\n    title=f\"TRAIN | min: {train_age.min()}, max: {train_age.max()}\",\n    ylabel=\"Age\"\n)\ntest_age = df_test['age']\nsns.distplot(test_age, ax=ax[2][1])\nmy_style_ax(\n    ax[2][1],\n    title=f\"TEST | min: {test_age.min()}, max: {test_age.max()}\"\n)\n\n# Body mass index\ntrain_bmi = df_train['bmi_kg/m^2'].fillna(0)\nsns.distplot(train_bmi, ax=ax[3][0])\nmy_style_ax(\n    ax[3][0],\n    title=f\"TRAIN | min: {train_bmi.min()}, max: {train_bmi.max()}\",\n    ylabel=\"Body mass ind.\"\n)\ntest_bmi = df_test['bmi_kg/m^2'].fillna(0)\nsns.distplot(test_bmi, ax=ax[3][1])\nmy_style_ax(\n    ax[3][1],\n    title=f\"TEST | min: {test_bmi.min()}, max: {test_bmi.max()}\"\n)\n\n# literality\ntrain_lit = df_train['laterality']\nsns.countplot(train_lit, ax=ax[4][0])\nmy_style_ax(\n    ax[4][0],\n    title=f\"TRAIN | {train_lit.value_counts()['Right']} right / {train_lit.value_counts()['Left']} left\",\n    ylabel='Literaluty'\n)\ntest_lit = df_test['laterality']\nsns.countplot(test_lit, ax=ax[4][1])\nmy_style_ax(\n    ax[4][1],\n    title=f\"TEST | {test_lit.value_counts()['Right']} right / {test_lit.value_counts()['Left']} left\"\n)\n\n# race\ntrain_r = df_train['race']\nsns.countplot(train_r, ax=ax[5][0])\nmy_style_ax(\n    ax[5][0],\n    title=f\"TRAIN | {train_r.value_counts()['White']} white / {train_r.value_counts()['Black or African American']} black\",\n    ylabel='Race'\n)\ntest_r = df_test['race']\nsns.countplot(test_r, ax=ax[5][1])\nmy_style_ax(\n    ax[5][1],\n    title=f\"TEST | {test_r.value_counts()['White']} white / {test_r.value_counts()['Black or African American']} black\"\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id='6'>Glomerulus cells EDA</a>"},{"metadata":{"id":"AuSneWLG_LMe","outputId":"628a4aa8-51d2-4afb-aceb-0461bf4a273f","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"_, axs = plt.subplots(nrows=6, ncols=1, figsize=(7, 33))\naxs = axs.flatten()\n\n# gl. avg area\ngl_avg = df_train['avg_g_area'] / 10**6\ngl_avg_min_ind, gl_avg_max_ind = np.argmin(gl_avg), np.argmax(gl_avg)\nsns.distplot(gl_avg, ax=axs[0])\nmy_style_ax(\n    axs[0],\n    title=f\"min: {np.round(gl_avg[gl_avg_min_ind], 3)}({df_train['image_file'][gl_avg_min_ind]}), \"\n          f\"max: {np.round(gl_avg[gl_avg_max_ind], 3)}({df_train['image_file'][gl_avg_max_ind]})\",\n    ylabel='avg_g_area * 10^-6'\n)\n# gl. count\ngl_count = df_train['glom_num']\ngl_count_min_ind, gl_count_max_ind = np.argmin(gl_count), np.argmax(gl_count)\nsns.distplot(gl_count, ax=axs[1])\nmy_style_ax(\n    axs[1],\n    title=f\"min: {np.round(gl_count[gl_count_min_ind], 3)}({df_train['image_file'][gl_count_min_ind]}), \"\n          f\"max: {np.round(gl_count[gl_count_max_ind], 3)}({df_train['image_file'][gl_count_max_ind]})\",\n    ylabel='glom_num'\n)\n# gl. content on cortex pixels\ngl_cp = df_train['glor_cells_on_pix']\ngl_cp_min_ind, gl_cp_max_ind = np.argmin(gl_cp), np.argmax(gl_cp)\nsns.distplot(gl_cp, ax=axs[2])\nmy_style_ax(\n    axs[2],\n    title=f\"min: {np.round(gl_cp[gl_cp_min_ind], 3)}({df_train['image_file'][gl_cp_min_ind]}), \"\n          f\"max: {np.round(gl_cp[gl_cp_max_ind], 3)}({df_train['image_file'][gl_cp_max_ind]})\",\n    ylabel='glor_cells_on_pix'\n)\n# sex\nfem = df_train[df_train['sex'] == 'Female']['glor_cells_on_pix']\nmale = df_train[df_train['sex'] == 'Male']['glor_cells_on_pix']\nsns.distplot(fem, ax=axs[3], label='Female')\nsns.distplot(male, ax=axs[3], label='Male')\nmy_style_ax(axs[3], \n            title=f'mean male: {round(male.mean(), 4)}, mean female: {round(fem.mean(), 4)}',\n            ylabel='sex \\ glor_cells_on_pix'\n           )\naxs[3].legend()\n# race\nrw = df_train[df_train['race'] == 'White']['glor_cells_on_pix']\nrb = df_train[df_train['race'] != 'White']['glor_cells_on_pix']\nsns.distplot(rw, ax=axs[4], label='White')\nsns.distplot(rb, ax=axs[4], label='Black')\nmy_style_ax(\n    axs[4],\n    title=f'mean white: {round(rw.mean(), 4)}, mean black: {round(rb.mean(), 4)}',\n    ylabel='race \\ glor_cells_on_pix'\n)\naxs[4].legend()\n# laterality\nlr = df_train[df_train['laterality'] == 'Right']['glor_cells_on_pix']\nll = df_train[df_train['laterality'] == 'Left']['glor_cells_on_pix']\nsns.distplot(lr, ax=axs[5], label='Right')\nsns.distplot(ll, ax=axs[5], label='Left')\nmy_style_ax(\n    axs[5],\n    title=f'mean right: {round(lr.mean(), 4)}, mean left: {round(ll.mean(), 4)}',\n    ylabel='laterality \\ glor_cells_on_pix'\n)\naxs[5].legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- # <a id='6.1'>Correlations(pearson)</a>"},{"metadata":{"id":"XkESWiIq1Ca1","outputId":"592b91e0-6fa7-491a-ac3e-c2034db31df0","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# correlations with glor_cells_on_pix / glor_num\n\ncols_num = ['age', 'avg_g_area', 'glom_num', \n        'glor_cells_on_pix', 'bmi_kg/m^2', 'height_centimeters', 'weight_kilograms']\nsns.heatmap(df_train[cols_num].corr(), annot=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- # <a id='6.2'>Regplot(glomerulus cells on pixel / bmi_kg/m^2(corr ~-0.96)</a>"},{"metadata":{"id":"6dLvMpvh-aQy","outputId":"cd725dd7-05bb-4a25-87b7-5d0a8740dce8","_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"df_glor_bmi_reg = df_train[['glor_cells_on_pix', 'bmi_kg/m^2']].dropna()\nsns.regplot(x='glor_cells_on_pix', y='bmi_kg/m^2', data=df_glor_bmi_reg)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}