{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport pdb\nimport glob\nimport pytz\nimport warnings\nimport pickle\nimport random\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm_notebook as tqdm\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, ExponentialLR\nfrom sklearn.model_selection import KFold\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nimport torch.backends.cudnn as cudnn\nfrom torch.utils.data import DataLoader, Dataset, sampler\nfrom albumentations.pytorch import ToTensorV2\nimport segmentation_models_pytorch as smp\n\nimport tifffile as tiff\nimport rasterio\nfrom rasterio.windows import Window","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**EDA(Exploratory Data Analysis):**","metadata":{}},{"cell_type":"code","source":"dset_info = pd.read_csv('/kaggle/input/hubmap-kidney-segmentation/HuBMAP-20-dataset_information.csv')\ndset_info.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dset_train=pd.read_csv('../input/hubmap-kidney-segmentation/train.csv')\ndset_train.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dset_submission=pd.read_csv('../input/hubmap-kidney-segmentation/sample_submission.csv')\ndset_submission.head(3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training Images\ntrain_files = sorted(glob.glob(os.path.join('../input/hubmap-kidney-segmentation', 'train/*.tiff')))\nprint(f'Number of training images: {len(train_files)}')\nprint('\\n'.join(train_files))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Test Images\ntest_files = sorted(glob.glob(os.path.join('../input/hubmap-kidney-segmentation', 'test/*.tiff')))\nprint(f'Number of test images: {len(test_files)}')\nprint('\\n'.join(test_files))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tifffile\nimport gc\n#train and test image analysis of height and weight distribution,where the formats vary(H,W,C) or(C,H,W) or(ndim,C,H,W)\n\nfor f in train_files[:2]:\n    image = tifffile.imread(f)\n    print(f'Image {f} shape: {image.shape}', flush=True)\n    del image\n    gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor f in test_files[:3]:\n    image = tifffile.imread(f)\n    print(f'Image {f} shape: {image.shape}', flush=True)\n    del image\n    gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Both train and test images vary in sizes\ndf_info = pd.read_csv(os.path.join('../input/hubmap-kidney-segmentation','HuBMAP-20-dataset_information.csv'))\ndf_info.head(4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#The size of the images varies greatly as well.\nimport matplotlib.pyplot as plt\nplt.scatter(df_info['width_pixels'], df_info['height_pixels'])\nplt.title('Image Height and Width')\nplt.xlabel('Width')\nplt.ylabel('Height')\nplt.xlim(0, df_info['width_pixels'].max() * 1.1)\nplt.ylim(0, df_info['height_pixels'].max() * 1.1)\nplt.grid()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#EDA on Glomeruli in test images\n\ndf_submit=pd.read_csv('../input/hubmap-submission-file/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Image Utility Functions**","metadata":{}},{"cell_type":"code","source":"plot_full_image = True\n\n# Number of glomeruli to display for each image\nnum_glom_display = 5\n\n# Number of glomberuli to save as tiff files.\nnum_glom_save = 5\n\nglob_scale = 0.25","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_to_image(rle_mask, image_shape):\n    \"\"\"\n    Converts an rle string to an image represented as a numpy array.\n    Reference: https://www.kaggle.com/paulorzp/rle-functions-run-lenght-encode-decode\n\n    :param rle_mask: string with rle mask.\n    :param image_shape: (width, height) of array to return\n    :return: Image as a numpy array. 1 = mask, 0 = background.\n    \"\"\"\n\n    # Processing\n    s = rle_mask.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    image = np.zeros(image_shape[0] * image_shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        image[lo:hi] = 1\n\n    return image.reshape(image_shape).T\n\ndef overlay_image_mask(image, mask, mask_color=(0,255,0), alpha=1.0):\n    im_f= image.astype(np.float32)\n#     if mask.ndim == 2:\n#         mask = np.expand_dims(mask,-1)        \n    mask_col = np.expand_dims(np.array(mask_color)/255.0, axis=(0,1))\n    return (im_f + alpha * mask * (np.mean(0.8 * im_f + 0.2 * 255, axis=2, keepdims=True) * mask_col - im_f)).astype(np.uint8)\n\ndef overlay_image_mask_original(image, mask, mask_color=(0,255,0), alpha=1.0):\n    return  np.concatenate((image, overlay_image_mask(image, mask)), axis=1)\n\ndef get_image_id(image_file):\n    return os.path.splitext(os.path.split(image_file)[1])[0]\n\n\ndef read_image(image_file, scale=1.0):\n    image = tifffile.imread(image_file).squeeze()\n    if image.shape[0] == 3:\n        image = np.transpose(image, (1,2,0))\n    \n    orig_shape = image.shape\n    if scale != 1.0:\n        image = cv2.resize(image, (0,0), fx=scale, fy=scale)\n    return image, orig_shape\n\ndef read_mask(image_file, image_shape, scale=1.0):\n    image_id = get_image_id(image_file)\n    train_info = dset_train.loc[dset_train['id'] == image_id]\n    submit_info = df_submit.loc[df_submit['id'] == image_id]\n    rle = train_info['encoding'].values[0] if len(train_info) > 0 else None\n    rle_test = submit_info['predicted'].values[0] if len(submit_info) > 0 else None\n    if rle is not None:\n        mask = rle_to_image(rle, (image_shape[1], image_shape[0]))\n        if scale != 1.0:\n            mask = cv2.resize(mask, (0,0), fx=scale, fy=scale)\n        return np.expand_dims(mask,-1)\n    elif rle_test is not None:\n        mask = rle_to_image(rle_test, (image_shape[1], image_shape[0]))\n        if scale != 1.0:\n            mask = cv2.resize(mask, (0,0), fx=scale, fy=scale)\n        return np.expand_dims(mask,-1)\n        \n    else:\n        return None       \n    \ndef read_image_mask(image_file, scale=1.0):\n    image, image_shape = read_image(image_file, scale)\n    mask = read_mask(image_file, image_shape, scale)\n    return image, mask\n\n\ndef get_tile(image, mask, x, y, tile_size, scale=1.0):\n    x = round(x * scale)\n    y = round(y * scale)\n    size = int(round(tile_size / 2 * scale))\n    image_s = image[y-size:y+size, x-size:x+size, :] \n    mask_s = mask[y-size:y+size, x-size:x+size, :]\n    return image_s, mask_s  \ndef get_particles(mask, scale=1.0):\n    num, labels, stats, centroids = cv2.connectedComponentsWithStats(mask)\n    df_particles = pd.DataFrame(dict(zip(['x','y','left','top','width','height','area'],\n                               [(centroids[1:,0]) / scale,\n                                (centroids[1:,1]) / scale,\n                                (stats[1:,cv2.CC_STAT_LEFT]) / scale,\n                                (stats[1:,cv2.CC_STAT_TOP]) / scale,\n                                (stats[1:,cv2.CC_STAT_WIDTH]) / scale,\n                                (stats[1:,cv2.CC_STAT_HEIGHT]) / scale,\n                                (stats[1:,cv2.CC_STAT_AREA]) / (scale * scale)])))\n    df_particles.sort_values(['x','y'], inplace=True, ignore_index=True)\n    df_particles['no'] = range(len(df_particles))\n    return df_particles\n\ndef analyze_image(image_file):\n    image_id = get_image_id(image_file)\n    image, image_shape = read_image(image_file, glob_scale)\n    mask = read_mask(image_file, image_shape, glob_scale)\n    mask_full = read_mask(image_file, image_shape, scale=1.0)\n    df_glom = get_particles(mask_full, scale=1.0)\n    df_glom['id'] = image_id\n    del mask_full\n    gc.collect()\n    \n    info = df_info[df_info['image_file'] == f'{image_id}.tiff']\n    print(f'Image ID:        {image_id:}')\n    print(f'Image Size:      {info[\"width_pixels\"].values[0]} x {info[\"height_pixels\"].values[0]}')\n    print(f'Patient No:      {info[\"patient_number\"].values[0]}')\n    print(f'Sex:             {info[\"sex\"].values[0]}')\n    print(f'Age:             {info[\"age\"].values[0]}')\n    print(f'Race:            {info[\"race\"].values[0]}')\n    print(f'Height:          {info[\"height_centimeters\"].values[0]} cm')\n    print(f'Weight:          {info[\"weight_kilograms\"].values[0]} kg')\n    print(f'BMI:             {info[\"bmi_kg/m^2\"].values[0]} kg/m^2')\n    print(f'Laterality:      {info[\"laterality\"].values[0]}')\n    print(f'Percent Cortex:  {info[\"percent_cortex\"].values[0]} %')\n    print(f'Percent Medulla: {info[\"percent_medulla\"].values[0]} %')\n    \n    # Plot full image\n    if plot_full_image:\n        scale = 0.1\n        image_small = cv2.resize(image, (0,0), fx=scale, fy=scale)\n        mask_small = cv2.resize(mask, (0,0), fx=scale, fy=scale)\n        mask_small = np.expand_dims(mask_small,-1) \n    \n        plt.figure(figsize=(16, 16))\n        plt.imshow(overlay_image_mask(image_small, mask_small))\n        plt.axis('off')\n\n    # Plot glomeruli images\n    fig_cols = 5\n    fig_rows = int(math.ceil(num_glom_display/fig_cols))\n    plt.figure(figsize=(4 * fig_cols, 4 * fig_rows))\n    if num_glom_save > 0 and not os.path.exists(image_id):\n        os.mkdir(image_id)\n    for i in range(min(max(num_glom_display, num_glom_save), len(df_glom))):\n        image_s, mask_s = get_tile(image,mask, df_glom['x'][i], df_glom['y'][i], 1000, scale=glob_scale)\n        \n        ovl = overlay_image_mask(image_s, mask_s)\n        if i < num_glom_display:\n            plt.subplot(fig_rows, fig_cols, i+1)\n            plt.imshow(ovl)\n            plt.axis('off')\n        if i < num_glom_save:\n            cv2.imwrite(f'{image_id}_{i:03}.png', cv2.cvtColor(ovl, cv2.COLOR_RGB2BGR))    \n    \n    del image, mask\n    gc.collect()\n    return df_glom","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\n#training images with glomerulis\ndf_glom = pd.DataFrame()\ndf_glom = df_glom.append(analyze_image(train_files[0]), ignore_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_glom = df_glom.append(analyze_image(train_files[1]), ignore_index=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_glom = df_glom.append(analyze_image(train_files[3]), ignore_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Basic Statistics of glomerulis in train images**","metadata":{}},{"cell_type":"code","source":"df_glom.to_csv('./glomeruli_train.csv')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_glom=pd.read_csv('./glomeruli_train.csv')\ndf_glom.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plotting  for 3 images number of glomeruli\ng = df_glom.groupby('id')\nplt.bar(g.size().index, g.size().values)\nplt.title('Number of Glomerulis in Image')\nplt.xticks(rotation=90)\nplt.grid()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"EDA On Test Images","metadata":{}},{"cell_type":"code","source":"df_glom_test = pd.DataFrame()\ndf_glom_test = df_glom_test.append(analyze_image(test_files[0]), ignore_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_glom_test = df_glom_test.append(analyze_image(test_files[1]), ignore_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_glom_test = df_glom_test.append(analyze_image(test_files[2]), ignore_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_glom_test = df_glom_test.append(analyze_image(test_files[3]), ignore_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_glom_test = df_glom_test.append(analyze_image(test_files[4]), ignore_index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_glom_test.to_csv('./glomeruli_test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To plot number of glomeruli per test images\ng = df_glom_test.groupby('id')\nplt.bar(g.size().index, g.size().values)\nplt.title('Number of Glomerulis in Image')\nplt.xticks(rotation=90)\nplt.grid()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To display glomerulis by size\ndf_glom_test.sort_values('area', inplace=True)\ndf_glom_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef plot_glom(df, image_id, glom_no):\n    #image, mask = read_image_mask(os.path.join(base_path, f'train/{image_id}.tiff'), scale=glob_scale)\n    image, mask = read_image_mask(os.path.join('../input/hubmap-kidney-segmentation', f'test/{image_id}.tiff'), scale=glob_scale)\n    glom = df.loc[(df['id'] == image_id) & (df['no'] == glom_no)]\n    im, ma = get_tile(image, mask, glom['x'].iloc[0], glom['y'].iloc[0], 1000, scale=glob_scale)\n    del image, mask\n    gc.collect()\n    plt.figure(figsize=(16,8))\n    plt.imshow(overlay_image_mask_original(im, ma))\n    plt.title(f'Image: {image_id}, Glomeruli No: {glom_no}, Area: {glom[\"area\"].iloc[0]}')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#To display 10 small glomerulis in test images\nfor i in range(10):\n    plot_glom(df_glom_test, df_glom_test['id'].iloc[i], df_glom_test['no'].iloc[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#To display largest glomerulis in test images\nfor i in range(len(df_glom_test)-5, len(df_glom_test)):\n    plot_glom(df_glom_test, df_glom_test['id'].iloc[i], df_glom_test['no'].iloc[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}