{"cells":[{"metadata":{"trusted":false,"_uuid":"ac75334d056420c2ce5a1a8db6dcaa20c588675e"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np  \nfrom pathlib import Path  \nimport h5py\nimport hashlib\nfrom tqdm import tqdm_notebook as tqdm\n\nPATH = Path('.')\nDATA_PATH = PATH/\"HisCancer\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"00f9ee7707f45c9741a59fe3b3936dbe1244ced7"},"cell_type":"markdown","source":"# This is a leak from the original dataset\nSee discussion: https://www.kaggle.com/c/histopathologic-cancer-detection/discussion/85424\nSome of the codes are from: https://www.kaggle.com/c/histopathologic-cancer-detection/discussion/85283\n\nHow did I discover this:\n - Carefully watch LB people see if anybody discovered a leak that boost them up with only a few submissions.\n - Learn from bestfiting's solution for HPA competition\n - I was trying to reproduce finding WSI ids in https://www.kaggle.com/c/histopathologic-cancer-detection/discussion/85283\n - Trying to develop my postprocess technique (inspired from TGS Salt)\n"},{"metadata":{"_uuid":"8d3a98b66aff36ed87d4f260fdd62e8e1f15c9a3"},"cell_type":"markdown","source":"## Create some functions to convert csv files to dict()"},{"metadata":{"trusted":false,"_uuid":"6654a1dffc937824abcc7291c6d03c022347fbd2"},"cell_type":"code","source":"def create_pcam16_hash_table(dset_img):\n    # creates a dictionary where each key is the sha-1 hash of the image data and the value is the index in the h5 file\n    hash_table = {}\n    for h5_id,img_id in enumerate(dset_img):\n        hash_table[hashlib.sha1(img_id).hexdigest()] = h5_id\n    return hash_table\n\ndef create_labels_dict(f):\n    # creates a dictionary where each key is the img_id and the value is tumor or not\n    df = pd.read_csv(f)\n    labels_dict = {}\n    for idx in range(len(df)):\n        img_id = df.iloc[idx,0]\n        is_tumor = df.iloc[idx,1]\n        labels_dict[img_id] = int(is_tumor)\n    return labels_dict\n\ndef create_wsi_dict(f):\n    # creates a dictionary where each key is the h5 index and the value is the wsi\n    df = pd.read_csv(f)\n    \n    dict_list = []\n    for y,x,tp,ctp,wsi in zip(df.coord_y.tolist(), df.coord_x.tolist(), df.tumor_patch.tolist(), df.center_tumor_patch.tolist(), df.wsi.tolist()):\n        dict_list.append({'coord_y':y,'coord_x':x, 'tumor_patch':tp,'center_tumor_patch':ctp,'wsi':wsi})\n    wsi_dict = dict(zip(df.index.tolist(),dict_list))\n    return wsi_dict","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b4a5faafeaf626cbb0b97266424ced8d46f98fbf"},"cell_type":"markdown","source":"## Create a function that generate all the information from the leak that may be helpful later\nI did not know how big the leak was at this point."},{"metadata":{"trusted":false,"_uuid":"b77f828c69f0fd4298a5fb34db4c08ef5f132a92"},"cell_type":"code","source":"def postprocess(h5_file, labels_file, meta_file, name='new'):\n    fimg = h5py.File(DATA_PATH/h5_file, 'r')\n\n    hash_table = create_pcam16_hash_table(fimg['x'])\n    labels_dict = create_labels_dict(DATA_PATH/labels_file)\n    wsi_dict = create_wsi_dict(DATA_PATH/meta_file)\n\n    ids = []\n    label = []\n    \n    y = []\n    x = []\n    tp = []\n    ctp = []\n    wsi = []\n\n    error_count = 0\n    pbar = tqdm(labels_dict.items())\n    for img_id, is_tumor in pbar:\n        if img_id in hash_table.keys():\n            h5_idx = hash_table[img_id]\n\n            ids.append(img_id)\n            label.append(is_tumor)\n\n            y.append(wsi_dict[h5_idx]['coord_y'])\n            x.append(wsi_dict[h5_idx]['coord_x'])\n            tp.append(wsi_dict[h5_idx]['tumor_patch'])\n            ctp.append(wsi_dict[h5_idx]['center_tumor_patch'])\n            wsi_id = wsi_dict[h5_idx]['wsi']\n            wsi.append(wsi_id)\n\n            pbar.set_description(\"[ERROR:{}] h5:{}, wsi_id:{}, id:{}, is_tumor:{}\".format(error_count, h5_idx, wsi_id, img_id, is_tumor))\n\n        else:\n            error_count = error_count+1\n            pbar.set_description(\"[ERROR:{}] Image {} does not exist in external file\".format(error_count, img_id))\n            \n    \n    df_full_wsi = pd.DataFrame({'id':ids,'coord_y':y,'coord_x':x, 'tumor_patch':tp,'center_tumor_patch':ctp,'wsi':wsi,'is_tumor':label},columns=['id', 'coord_y', 'coord_x', 'tumor_patch', 'center_tumor_patch', 'wsi','is_tumor'])\n    df_full_wsi.to_csv(DATA_PATH/labels_file.replace('.csv', '_wsi_{}.csv'.format(name)),index=False)\n    \n    print(\"Out of {} images, {} is unknown\".format(len(pbar), error_count))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e03691538c3c257c308c32c6bb6e01191134a091"},"cell_type":"markdown","source":"## Simply run the function above in `train`, `valid`, and `test`"},{"metadata":{"trusted":false,"_uuid":"e5cee273da7c9ca0302b2379112269a1f9267f09"},"cell_type":"code","source":"postprocess('camelyonpatch_level_2_split_train_x.h5', \"train_labels.csv\", 'camelyonpatch_level_2_split_train_meta.csv', name='train')","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"16b34b9e29c7a1f22d3889e7613545b4551a8171"},"cell_type":"code","source":"postprocess('camelyonpatch_level_2_split_valid_x.h5', \"sample_submission.csv\", 'camelyonpatch_level_2_split_valid_meta.csv', name='valid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"78085e021f5d43610526ac47025549b1f9836008"},"cell_type":"code","source":"postprocess('camelyonpatch_level_2_split_test_x.h5', \"sample_submission.csv\", 'camelyonpatch_level_2_split_test_meta.csv', name='test')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b30ed546554c71fa377d5545ed54f233d0e9c91"},"cell_type":"markdown","source":"## Test if the leak has real effect on LB\nAfter looking at the files I generated above, I realized `tumor_patch` might be useful:\n 1. I know that if the image is not tumor_patch, then no cancer should be detected in the sliced images"},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"931c3301288d2e896d1178a47ad16aff7bd1aaed"},"cell_type":"code","source":"origin = pd.read_csv(DATA_PATH/'000c46d3-CP4_F1_PT2019-02-26-06-23-29-725832-two_VT000c46d3_LR0.01_BS64_IMG224.pth-two-F0-T0.5-Prob-Prep(240)TTAPowerNasnet0.9714.csv').set_index('Id')\norigin_dict = origin.to_dict()\n\nwsi_valid = pd.read_csv(DATA_PATH/'sample_submission_wsi_valid.csv').set_index('id')\nwsi_valid_dict = wsi_valid.to_dict()\n\nwsi_test = pd.read_csv(DATA_PATH/'sample_submission_wsi_test.csv').set_index('id')\nwsi_test_dict = wsi_test.to_dict()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f792d8ba8409d93a6c893d0798cbc04a230de867"},"cell_type":"code","source":"# Test Code\n# origin_dict['Label']\nwsi_valid_dict","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15fb813c43cc2f90c81e48309a633b5414a1c1b2"},"cell_type":"markdown","source":"## So I postprocessed my highest scored submission."},{"metadata":{"trusted":false,"_uuid":"cf52020b10f846b3f9885c5bb7fb8a7c67f51c2f"},"cell_type":"code","source":"pbar = tqdm(origin_dict['Label'].keys())\n\nimg_id_new = []\nlabel_new = []\n\nerror_count = 0\nfor img_id in pbar:\n    if img_id in wsi_valid_dict['tumor_patch'].keys():\n        if wsi_valid_dict['tumor_patch'][img_id] == False:\n            img_id_new.append(img_id)\n            label_new.append(0.)\n            pbar.set_description(\"[ERROR={}] {} = 0\".format(error_count, img_id))\n        else:\n            img_id_new.append(img_id)\n            label_new.append(origin_dict['Label'][img_id])\n    else:\n        img_id_new.append(img_id)\n        label_new.append(origin_dict['Label'][img_id])\n        error_count = error_count+1\n        pbar.set_description(\"[ERROR={}] No id found.\".format(error_count))\n\nout = pd.DataFrame({'id': img_id_new, 'label': label_new}, columns=['id', 'label'])\nout.to_csv(DATA_PATH/'output.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cdc2f74ebf19e25c5fdb8788a461bd43e133b604"},"cell_type":"markdown","source":"## Wow, negative labels in valid set really work, how about positive labels? Will it affect the private LB?\nIt will affect private LB since my score stay exactly the same with only private LB postprocessed."},{"metadata":{"trusted":false,"_uuid":"9b891778b9726dc17a7579575a2ffd5aa227039c"},"cell_type":"code","source":"pbar = tqdm(origin_dict['Label'].keys())\n\nimg_id_new = []\nlabel_new = []\n\nerror_count = 0\nfor img_id in pbar:\n    if img_id in wsi_test_dict['tumor_patch'].keys():\n        if wsi_test_dict['tumor_patch'][img_id] == False:\n            img_id_new.append(img_id)\n            label_new.append(0.)\n            pbar.set_description(\"[ERROR={}] {} = 0\".format(error_count, img_id))\n        else:\n            img_id_new.append(img_id)\n            label_new.append(origin_dict['Label'][img_id])\n    else:\n        img_id_new.append(img_id)\n        label_new.append(origin_dict['Label'][img_id])\n        error_count = error_count+1\n        pbar.set_description(\"[ERROR={}] No id found.\".format(error_count))\n\nout = pd.DataFrame({'id': img_id_new, 'label': label_new}, columns=['id', 'label'])\nout.to_csv(DATA_PATH/'output.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cacb105f97fd24fb4d1ed4e91a578520bc9840de"},"cell_type":"markdown","source":"## Make a perfect cheatsheet"},{"metadata":{"trusted":false,"_uuid":"7ddd4b0683e3677ce4960ff2828525161b3fe5b2"},"cell_type":"code","source":"pbar = tqdm(origin_dict['Label'].keys())\n\nimg_id_new = []\nlabel_new = []\n\nerror_count = 0\nfor img_id in pbar:\n    if img_id in wsi_test_dict['tumor_patch'].keys():\n        if wsi_test_dict['center_tumor_patch'][img_id] == False:\n            img_id_new.append(img_id)\n            label_new.append(1)\n            pbar.set_description(\"[ERROR={}] {} = 0\".format(error_count, img_id))\n        elif wsi_test_dict['center_tumor_patch'][img_id] == True:\n            img_id_new.append(img_id)\n            label_new.append(0)\n            pbar.set_description(\"[ERROR={}] {} = 0\".format(error_count, img_id))\n        else:\n            img_id_new.append(img_id)\n            label_new.append(origin_dict['Label'][img_id])\n            error_count = error_count+1\n    elif img_id in wsi_valid_dict['tumor_patch'].keys():\n        if wsi_valid_dict['center_tumor_patch'][img_id] == False:\n            img_id_new.append(img_id)\n            label_new.append(0)\n            pbar.set_description(\"[ERROR={}] {} = 0\".format(error_count, img_id))\n        elif wsi_valid_dict['center_tumor_patch'][img_id] == True:\n            img_id_new.append(img_id)\n            label_new.append(1)\n            pbar.set_description(\"[ERROR={}] {} = 0\".format(error_count, img_id))\n        else:\n            img_id_new.append(img_id)\n            label_new.append(origin_dict['Label'][img_id])\n            error_count = error_count+1\n    else:\n        img_id_new.append(img_id)\n        label_new.append(origin_dict['Label'][img_id])\n        error_count = error_count+1\n        pbar.set_description(\"[ERROR={}] No id found.\".format(error_count))\n\nout = pd.DataFrame({'id': img_id_new, 'label': label_new}, columns=['id', 'label'])\nout.to_csv(DATA_PATH/'output.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0704ec861e8dc558bff249f24d33dc66f6e85696"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}