{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Contents:\n* <h1 style=\"padding: 1rem;\n          color:black;\n          text-align:left;\n          margin:0 auto;\n          font-size:1.5rem;\"><a href=\"#introduction\">Dataset Info</a></h1>\n* <h1 style=\"padding: 1rem;\n          color:black;\n          text-align:left;\n          margin:0 auto;\n          font-size:1.5rem;\"><a href=\"#statistics\">Dataset Statistics</a></h1>\n* <h1 style=\"padding: 1rem;\n          color:black;\n          text-align:left;\n          margin:0 auto;\n          font-size:1.5rem;\"><a href=\"#visualization\">Visualization</a></h1>\n* <h1 style=\"padding: 1rem;\n          color:black;\n          text-align:left;\n          margin:0 auto;\n          font-size:1.5rem;\"><a href=\"#references\">References</a></h1>\n","metadata":{}},{"cell_type":"markdown","source":"<div class=\"alert alert-info\" role=\"alert\">\n<h1 style=\"padding: 2rem;\n           color:black;\n           text-align:center;\n           margin:0 auto;\n           font-size:2rem;\">Dataset Info</h1>\n</div>","metadata":{}},{"cell_type":"markdown","source":"> ### Test/Train data contains 7/10 columns\n*  id - Identification number of an image\n*  organ - Indicates the organ\n*  data_source - Source of an image (HPA or Hubmap)\n*  img_height - The image height in pixels\n*  img_width - The image width in pixels\n*  pixel_size - Indicaties a single pixel's width and height in µm (micrometers)\n*  tissue_thickness - Indicaties thickness of a tissue in µm (micrometers)","metadata":{}},{"cell_type":"markdown","source":"> ### Additional columns in Train\n*  rle - Run-length encoded annotation\n*  age - The age of the patient\n*  sex - The gender of the patient","metadata":{}},{"cell_type":"markdown","source":"<div id=10 style=\"color:white;    \n           display:fill;\n           border-radius:5px;\n           font-size:110%;\n           background-color:#3EC70B;\n           font-family:Verdana;\n           letter-spacing:1px;\n           display:flex;\n           justify-content:center;\n           border-width: 0.25rem;\n           border-style: solid;\n           border-color: #256D85\">","metadata":{}},{"cell_type":"markdown","source":"<div class=\"alert alert-warning\" role=\"alert\">\n<a id=\"statistics\"><h1 style=\"padding: 2rem;\n          color:black;\n          text-align:center;\n          margin:0 auto;\n          font-size:2rem;\">\n Dataset Statistics\n</h1></a>\n</div>","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport pandas_profiling \nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nimport pprint\nfrom termcolor import colored\nimport json\nimport cv2\nimport os\nos.listdir(\"../input\")\nwarnings.simplefilter('ignore')\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.554064Z","iopub.execute_input":"2022-08-24T13:22:42.554447Z","iopub.status.idle":"2022-08-24T13:22:42.566166Z","shell.execute_reply.started":"2022-08-24T13:22:42.554416Z","shell.execute_reply":"2022-08-24T13:22:42.564454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_ANNOTATIONS_PATH = \"../input/hubmap-organ-segmentation/train_annotations/\"\nTRAIN_IMG_PATH = \"../input/hubmap-organ-segmentation/train_images/\"\nTEST_IMG_PATH = \"../input/hubmap-organ-segmentation/test_images/\"\nTRAIN_CSV = \"../input/hubmap-organ-segmentation/train.csv\"\nTEST_CSV = \"../input/hubmap-organ-segmentation/test.csv\"\nSUBMISSION_CSV = \"../input/hubmap-organ-segmentation/sample_submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.569484Z","iopub.execute_input":"2022-08-24T13:22:42.570043Z","iopub.status.idle":"2022-08-24T13:22:42.578420Z","shell.execute_reply.started":"2022-08-24T13:22:42.569994Z","shell.execute_reply":"2022-08-24T13:22:42.576920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train RLE annotations defines the boundaries of the mask of each function tissue unit. Train data already have the copy of it as **rle** column.","metadata":{}},{"cell_type":"code","source":"with open(TRAIN_ANNOTATIONS_PATH + \"10044.json\") as f:\n    annotation = json.load(f)\n    print(\"Annotation RLE: {}\".format(annotation[0]))","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.581411Z","iopub.execute_input":"2022-08-24T13:22:42.582290Z","iopub.status.idle":"2022-08-24T13:22:42.593452Z","shell.execute_reply.started":"2022-08-24T13:22:42.582237Z","shell.execute_reply":"2022-08-24T13:22:42.591867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df = pd.read_csv(TRAIN_CSV)\ntest_data_df = pd.read_csv(TEST_CSV)\ntrain_data_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.595683Z","iopub.execute_input":"2022-08-24T13:22:42.596862Z","iopub.status.idle":"2022-08-24T13:22:42.763928Z","shell.execute_reply.started":"2022-08-24T13:22:42.596799Z","shell.execute_reply":"2022-08-24T13:22:42.762450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df.head(6)","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.766708Z","iopub.execute_input":"2022-08-24T13:22:42.767903Z","iopub.status.idle":"2022-08-24T13:22:42.790273Z","shell.execute_reply.started":"2022-08-24T13:22:42.767862Z","shell.execute_reply":"2022-08-24T13:22:42.789167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Submission file requires only **id** and **rle** columns.","metadata":{}},{"cell_type":"code","source":"submission_df = pd.read_csv(SUBMISSION_CSV)\nsubmission_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.791610Z","iopub.execute_input":"2022-08-24T13:22:42.791937Z","iopub.status.idle":"2022-08-24T13:22:42.809546Z","shell.execute_reply.started":"2022-08-24T13:22:42.791907Z","shell.execute_reply":"2022-08-24T13:22:42.808645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.811198Z","iopub.execute_input":"2022-08-24T13:22:42.811560Z","iopub.status.idle":"2022-08-24T13:22:42.821701Z","shell.execute_reply.started":"2022-08-24T13:22:42.811513Z","shell.execute_reply":"2022-08-24T13:22:42.820590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Data contains some images with different sizes. The minimum/maximum value of **img_height** and **img_width** is 2308/3070.","metadata":{}},{"cell_type":"code","source":"train_data_df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.823339Z","iopub.execute_input":"2022-08-24T13:22:42.823967Z","iopub.status.idle":"2022-08-24T13:22:42.863387Z","shell.execute_reply.started":"2022-08-24T13:22:42.823933Z","shell.execute_reply":"2022-08-24T13:22:42.861934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.865084Z","iopub.execute_input":"2022-08-24T13:22:42.866097Z","iopub.status.idle":"2022-08-24T13:22:42.896261Z","shell.execute_reply.started":"2022-08-24T13:22:42.866057Z","shell.execute_reply":"2022-08-24T13:22:42.895349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df[\"age\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.897463Z","iopub.execute_input":"2022-08-24T13:22:42.898318Z","iopub.status.idle":"2022-08-24T13:22:42.907086Z","shell.execute_reply.started":"2022-08-24T13:22:42.898282Z","shell.execute_reply":"2022-08-24T13:22:42.905573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.912444Z","iopub.execute_input":"2022-08-24T13:22:42.913637Z","iopub.status.idle":"2022-08-24T13:22:42.924238Z","shell.execute_reply.started":"2022-08-24T13:22:42.913584Z","shell.execute_reply":"2022-08-24T13:22:42.923322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<div class=\"alert alert-success\" role=\"alert\">\n<a id=\"visualization\"><h1 style=\"padding: 2rem;\n          color:black;\n          text-align:center;\n          margin:0 auto;\n          font-size:2rem;\">\n Visualization\n</h1></a>\n\n</div>","metadata":{}},{"cell_type":"code","source":"sns.countplot(x='sex', data=train_data_df, palette='Set3')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:42.925780Z","iopub.execute_input":"2022-08-24T13:22:42.926368Z","iopub.status.idle":"2022-08-24T13:22:43.104149Z","shell.execute_reply.started":"2022-08-24T13:22:42.926323Z","shell.execute_reply":"2022-08-24T13:22:43.103088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(train_data_df, x=\"age\", hue=\"age\", element=\"step\")","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:43.105729Z","iopub.execute_input":"2022-08-24T13:22:43.106465Z","iopub.status.idle":"2022-08-24T13:22:44.018891Z","shell.execute_reply.started":"2022-08-24T13:22:43.106419Z","shell.execute_reply":"2022-08-24T13:22:44.017376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(train_data_df['organ'],train_data_df['age'], hue=train_data_df['sex']);\ntrain_data_df['organ'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:44.021022Z","iopub.execute_input":"2022-08-24T13:22:44.022144Z","iopub.status.idle":"2022-08-24T13:22:44.518191Z","shell.execute_reply.started":"2022-08-24T13:22:44.022075Z","shell.execute_reply":"2022-08-24T13:22:44.517260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_cv2_edges(file_path, caption):\n    img = cv2.cvtColor(cv2.imread(file_path),cv2.COLOR_BGR2RGB)\n    canny_edge = cv2.Canny(image=img, threshold1=90, threshold2=100)\n    sobely64f = cv2.Sobel(src=img, ddepth=-1, dx=0, dy=1, ksize=5)\n\n    plt.figure(figsize=(20,20))\n    plt.subplot(1,3,1)\n    plt.imshow(img, cmap='gray')\n    plt.title('Original: {}'.format(caption)) \n    plt.axis(\"off\")\n\n    plt.subplot(1,3,2)\n    plt.imshow(canny_edge, cmap='gray')\n    plt.title('Canny Edge')\n    plt.axis(\"off\")\n\n    plt.subplot(1,3,3)\n    plt.imshow(sobely64f, cmap='gray')\n    plt.title('Sobel Edge')\n    plt.axis(\"off\")\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:44.519972Z","iopub.execute_input":"2022-08-24T13:22:44.520695Z","iopub.status.idle":"2022-08-24T13:22:44.530805Z","shell.execute_reply.started":"2022-08-24T13:22:44.520653Z","shell.execute_reply":"2022-08-24T13:22:44.529332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Test data contains only single row.","metadata":{}},{"cell_type":"code","source":"test_data_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:44.532340Z","iopub.execute_input":"2022-08-24T13:22:44.533604Z","iopub.status.idle":"2022-08-24T13:22:44.555230Z","shell.execute_reply.started":"2022-08-24T13:22:44.533545Z","shell.execute_reply":"2022-08-24T13:22:44.554002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Spleen visualization using basic edge detection algorithms.","metadata":{}},{"cell_type":"code","source":"show_cv2_edges(TEST_IMG_PATH + \"10078.tiff\", caption=\"Unknown\")","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:44.556884Z","iopub.execute_input":"2022-08-24T13:22:44.557241Z","iopub.status.idle":"2022-08-24T13:22:46.467082Z","shell.execute_reply.started":"2022-08-24T13:22:44.557199Z","shell.execute_reply":"2022-08-24T13:22:46.465445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Visualization of the organs from the train data.","metadata":{}},{"cell_type":"code","source":"organs_to_visualize = list()\nfor organ in train_data_df.organ.unique():\n    first_row = train_data_df.loc[train_data_df.organ == organ].head(1).reset_index()\n    organs_to_visualize.append(first_row)","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:46.468596Z","iopub.execute_input":"2022-08-24T13:22:46.468957Z","iopub.status.idle":"2022-08-24T13:22:46.485567Z","shell.execute_reply.started":"2022-08-24T13:22:46.468927Z","shell.execute_reply":"2022-08-24T13:22:46.484640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for organ in organs_to_visualize:\n    filename = str(organ.id._get_value(0))\n    caption = str(organ.organ._get_value(0))\n    show_cv2_edges(TRAIN_IMG_PATH + filename +\".tiff\", caption=caption)","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:22:46.486955Z","iopub.execute_input":"2022-08-24T13:22:46.488162Z","iopub.status.idle":"2022-08-24T13:23:04.768990Z","shell.execute_reply.started":"2022-08-24T13:22:46.488113Z","shell.execute_reply":"2022-08-24T13:23:04.767591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_mask(filename, caption): \n    def rle2mask(mask_rle, shape=(3000, 3000)):\n        '''\n        mask_rle: run-length as string formated (start length)\n        shape: (width,height) of array to return \n        Returns numpy array, 1 - mask, 0 - background\n\n        '''\n        s = mask_rle.split()\n        starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n        starts -= 1\n        ends = starts + lengths\n        img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n        for lo, hi in zip(starts, ends):\n            img[lo:hi] = 1\n        return img.reshape(shape).T\n    \n    file_path = TRAIN_IMG_PATH + filename +\".tiff\"\n    img = cv2.cvtColor(cv2.imread(file_path),cv2.COLOR_BGR2RGB)\n    mask = rle2mask(train_data_df[train_data_df['id']==int(filename)][\"rle\"].values[0], (img.shape[1], img.shape[0]))\n    \n    plt.figure(figsize=(15,15))\n    plt.imshow(img)\n    plt.imshow(mask, cmap=\"magma\", alpha=0.7)\n    plt.title('Original: {}'.format(caption)) \n    plt.axis(\"off\")\n","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:23:04.770430Z","iopub.execute_input":"2022-08-24T13:23:04.770810Z","iopub.status.idle":"2022-08-24T13:23:04.783055Z","shell.execute_reply.started":"2022-08-24T13:23:04.770777Z","shell.execute_reply":"2022-08-24T13:23:04.781568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Visualization of the organs from the train data with mask overlay.","metadata":{}},{"cell_type":"code","source":"for organ in organs_to_visualize:\n    filename = str(organ.id._get_value(0))\n    caption = str(organ.organ._get_value(0))\n    display_mask(filename, caption=caption)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:23:04.784989Z","iopub.execute_input":"2022-08-24T13:23:04.785868Z","iopub.status.idle":"2022-08-24T13:23:18.488527Z","shell.execute_reply.started":"2022-08-24T13:23:04.785818Z","shell.execute_reply":"2022-08-24T13:23:18.487168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n------------\n\n# <a id=\"references\">References</a>\n\n* https://www.kaggle.com/code/paulorzp/rle-functions-run-lenght-encode-decode/script","metadata":{"execution":{"iopub.status.busy":"2022-08-24T13:23:52.826877Z","iopub.execute_input":"2022-08-24T13:23:52.827370Z","iopub.status.idle":"2022-08-24T13:23:52.835436Z","shell.execute_reply.started":"2022-08-24T13:23:52.827335Z","shell.execute_reply":"2022-08-24T13:23:52.834103Z"}}}]}