{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n# HuBMAP and Portal Mapping","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"<h4> External data from portal.hubmapconsortium added to kaggle datasets is mapped to competition data here </h4>\n<h4> Initial assumption looks at matching image height and width to HuBMAP-20-dataset_information.csv </h4>\n<h4> Imagehash is used to consider if these pairs are likely matches </h4>\n<h4> File for output created that includes HuBMAP dataset information and flags file in test/train, and if FFPE </h4>","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/c/hubmap-kidney-segmentation/discussion/233336\n#  External data source ---HuBMAP website : https://portal.hubmapconsortium.org/search?entity_type[0]=Dataset\n\n# # in notebook add data by url for datasets \n# https://www.kaggle.com/narainp/hub-ext-2\n# https://www.kaggle.com/narainp/hubmap-ext\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imports\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\nimport tifffile, cv2, gc\nfrom pathlib import Path\n\nfrom PIL import Image\nimport imagehash\n\ngc.enable()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH1 = '../input/hubmap-ext'\nDATA_PATH2 = '../input/hub-ext-2'\n\nprint(f'No. of ext images : {len(os.listdir(DATA_PATH1))}')\nprint(f'No. of ext2 images: {len(os.listdir(DATA_PATH2))}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fnames1 = np.array(os.listdir(DATA_PATH1))\nfnames2 = np.array(os.listdir(DATA_PATH2))\nprint('ext images :',fnames1)\nprint('ext2 images:',fnames2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataset_information used for possible match by same height, width\ninfo = pd.read_csv('../input/hubmap-kidney-segmentation/HuBMAP-20-dataset_information.csv')\ninfo.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to check if image will belong to train or test\ntrain = pd.read_csv('../input/hubmap-kidney-segmentation/train.csv')\ntest =  pd.read_csv('../input/hubmap-kidney-segmentation/sample_submission.csv')\ntest.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path1 = Path(DATA_PATH1)\npath2 = Path(DATA_PATH2)\nhubpath = Path('../input/hubmap-kidney-segmentation/train')\nhubpathtst = Path('../input/hubmap-kidney-segmentation/test')\npath1, path2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hash(image):\n    if len(image.shape) == 5:\n        image = image.squeeze().transpose(1, 2, 0)\n    elif image.shape[0] == 3:\n        image = image.transpose(1, 2, 0)\n    #if image.shape[0] == 3:\n    #    image = image.transpose(1, 2, 0)\n    image = Image.fromarray(image)    \n    hash =  imagehash.average_hash(image)\n    del image\n    gc.collect()\n    return hash","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hash_hubmap(hubimg):    \n    if hubimg.split('.')[0] in test.id.values:\n        himage = tifffile.imread(hubpathtst/f\"{hubimg}\")\n    else:\n        himage = tifffile.imread(hubpath/f\"{hubimg}\")\n    if len(himage.shape) == 5:\n        himage = himage.squeeze().transpose(1, 2, 0)\n    elif himage.shape[0] == 3:\n        himage = himage.transpose(1, 2, 0)\n        \n    #if himage.shape[0] == 3:\n    #    himage = himage.transpose(1, 2, 0)\n    himage = Image.fromarray(himage)  \n    hash = imagehash.average_hash(himage)\n    del himage\n    gc.collect()\n    \n    return hash","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if images are too big notebook will exceed memory, for now these are skipped\nprint(fnames1[9], fnames2[2])  # testing showed these are too big for imagehash,   h x w 1731207120,  2025172800\nhwfails = 1731207120","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Match first external dataset","metadata":{}},{"cell_type":"code","source":"h1 = []\nw1 = []\nim1 = []\ntrn1 = []\nhash1 = []\nfor i in range(len(fnames1)):\n    image = tifffile.imread(path1/fnames1[i])  \n    # print(fnames1[i])  # testing\n    h1.append(image.shape[1])\n    w1.append(image.shape[2])\n    if (len(info.image_file[(info.height_pixels==image.shape[1]) &  (info.width_pixels==image.shape[2])])) > 0:\n        imfile = info.image_file[(info.height_pixels==image.shape[1]) &  (info.width_pixels==image.shape[2])].values[0]  \n        trn1.append(imfile.split('.')[0] in test.id.values)            \n        if (image.shape[1]*image.shape[2])>=hwfails :   #hwfails  h x w 1731207120  if i==9 known to fail\n            hash1.append('too big will fail')\n        else:    \n            ihash = get_hash(image)\n            del image\n            gc.collect()\n            hhash = get_hash_hubmap(imfile)\n            match = ihash==hhash \n            hash1.append(match)\n    else:\n        imfile = 'unknown'   \n        trn1.append('unknown')\n        hash1.append('unknown')\n    im1.append(imfile)          \n            \n   # del image\n    gc.collect()\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#h1, w1, im1,trn1, hash1   # to check","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hubportal = pd.DataFrame(\n                {          \n                'portal_file' : fnames1,\n                'p_height_pixels' : h1,\n                'p_width_pixels'  : w1,  \n                'p_in_test' : trn1,  \n                'p_hash_match' : hash1,    \n                'image_file' : im1   \n                })    \n#hubportal.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if portal file contains FFPE set flag is FFPE  \ndef check_4_ffpe(x):\n    return ('FFPE' in x.split('.')[0].split('_'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hubportal['is_FFPE'] =   hubportal['portal_file'].apply(lambda x:  check_4_ffpe(x ))\nhubportal['p_data_path'] = DATA_PATH1\nhubportal.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this is the one that crashes, may be alternatives that will work TBA \n#ppath = Path(hubportal.p_data_path[9])\n#pfname = hubportal.portal_file[9]\n#image = tifffile.imread(ppath/pfname)\n#if image.shape[0] == 3:\n#    image = image.transpose(1, 2, 0)\n#image = Image.fromarray(image)\n#hash = imagehash.average_hash(image)\n#print(hash)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Match second external dataset","metadata":{}},{"cell_type":"code","source":"h2 = []\nw2 = []\nim2 = []\ntrn2 = []\nhash2 = []\nfor i in range(len(fnames2)):\n   # print(fnames2[i])  # testing\n    image = tifffile.imread(path2/fnames2[i])    \n    h2.append(image.shape[1])\n    w2.append(image.shape[2])\n    if (len(info.image_file[(info.height_pixels==image.shape[1]) &  (info.width_pixels==image.shape[2])])) > 0:\n        imfile = info.image_file[(info.height_pixels==image.shape[1]) &  (info.width_pixels==image.shape[2])].values[0]\n        trn2.append(imfile.split('.')[0] in test.id.values) \n        if (image.shape[1]*image.shape[2])>=hwfails :   #hwfails  h x w 1731207120 #if i == 2: # h x w 2025172800\n            hash2.append('too big will fail')\n        else:  \n            ihash = get_hash(image)\n            del image\n            gc.collect()\n            hhash = get_hash_hubmap(imfile)\n            \n            match = ihash==hhash \n            hash2.append(match)\n    else:\n        imfile = 'unknown'   \n        trn2.append('unknown')\n        hash2.append('unknown')  \n    im2.append(imfile)      \n        \n    #del image\n    gc.collect()\n    \n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#h2,w2,im2,trn2,hash2 # to check","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hubportal2 = pd.DataFrame(\n                {          \n                'portal_file' : fnames2,\n                'p_height_pixels' : h2,\n                'p_width_pixels'  : w2, \n                'p_in_test' : trn2,    \n                'p_hash_match' : hash2,     \n                'image_file' : im2   \n                })    \nhubportal2.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hubportal2['is_FFPE'] =   hubportal2['portal_file'].apply(lambda x:  check_4_ffpe(x ))\nhubportal2['p_data_path'] = DATA_PATH2\nhubportal2.head(11)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge matches and dataset information to consolidate mapping ","metadata":{}},{"cell_type":"code","source":"hubportal_info = hubportal.append(hubportal2).reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hubportal_info= pd.merge(hubportal_info, info, how='left', on='image_file')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hubportal_info.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(hubportal_info[hubportal_info.width_pixels.isnull()]), hubportal_info.portal_file[hubportal_info.width_pixels.isnull()]\n# 2 in portal not matched \n# VAN0011-RK-3-10-PAS_registered.ome.tif   other matched VAN0011 is for patient 67177\n# VAN0003-LK-32-21-PAS_registered.ome.tif  other matched VAN0003 is for patient 65631","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info_images = info.image_file.values\nlen(info_images)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(info_images)):\n    if len(hubportal_info[hubportal_info.image_file== info_images[i]]) ==0:\n        print(info_images[i])\n# c68fe75ea.tiff only image not matched in hubmap_20_dataset_info for patient 67112    \n# other matched image for patient 67112 is VAN0010-LK-160-2-PAS_FFPE.ome.tif 2ec3f1bb9.tiff in test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(hubportal_info[hubportal_info.is_FFPE==1]),len(hubportal_info[hubportal_info.is_FFPE==0])  # is_FFPE ==1 9    is_FFPE ==0  12","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(hubportal_info[hubportal_info.p_in_test==True]) # all 5 test images mapped also","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save file ","metadata":{}},{"cell_type":"code","source":"hubportal_info.to_csv('hubmap_portal_mapping.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}