{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook will process original training data of the Histopathologic Cancer Dataset.\n\nObjectives:\n- avoid validation data being in the same WSI with the training data\n- remove known duplicates\n- remove redundant white crops\n- optional reducing of the dataset for debugging\n- prepare data for easy feeding into fastai\n\nOutput:\n- shuffled CSV file with 3 columns: id, label, isValidation\n- output file name: WSI_splitted_data.csv\n\nExamples of using the result CSV file is shown at the end of this notebook, including **fastai** example of **ImageDataLoaders.from_pd**. It maybe usefull for experimenting with different network architectures using the same dataset for comparison.\n\nAcknowledgement: \n\n  This work is integrating several ideas about improving quality of data from several contributors. Detailed acknowledgements and links are in the code below.","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n#from PIL import Image\n#import cv2\nimport matplotlib.pyplot as plt\n#import matplotlib.patches as patches\n%matplotlib inline\n\nimport random\n#import time\nimport sys\n\nfrom sklearn.utils import shuffle\nfrom sklearn.utils import resample\n#from sklearn.metrics import roc_auc_score\n\n#from tqdm import tqdm_notebook\n\n#import torch \n#import torch.nn as nn\n#import torch.nn.functional as F\n#import torchvision\n#import torchvision.transforms.functional as TF\n#from torchvision import models \n#from torchinfo import summary\n#from torch.utils.data import TensorDataset, DataLoader, Dataset\n\nimport warnings\nwarnings.simplefilter('ignore', category=FutureWarning)\nwarnings.simplefilter('ignore', category=UserWarning)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:42:25.170353Z","iopub.execute_input":"2022-05-22T11:42:25.170929Z","iopub.status.idle":"2022-05-22T11:42:25.666839Z","shell.execute_reply.started":"2022-05-22T11:42:25.170823Z","shell.execute_reply":"2022-05-22T11:42:25.665814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Seed = 6839\n\n# optionally reduce size of training and validation\n# this may help to dedug pre-run\nREDUCE = False #True\nreducedTrainingSize = 1024\nreducedValidationSize = 256","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:42:25.668552Z","iopub.execute_input":"2022-05-22T11:42:25.670321Z","iopub.status.idle":"2022-05-22T11:42:25.676203Z","shell.execute_reply.started":"2022-05-22T11:42:25.670280Z","shell.execute_reply":"2022-05-22T11:42:25.675090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=833):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    #torch.manual_seed(seed)\n\nseed_everything(Seed)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:42:25.677922Z","iopub.execute_input":"2022-05-22T11:42:25.678195Z","iopub.status.idle":"2022-05-22T11:42:25.689935Z","shell.execute_reply.started":"2022-05-22T11:42:25.678165Z","shell.execute_reply":"2022-05-22T11:42:25.688989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path='../input/histopathologic-cancer-detection/train/'\nannotation_file='../input/histopathologic-cancer-detection/train_labels.csv'","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:42:25.691470Z","iopub.execute_input":"2022-05-22T11:42:25.691851Z","iopub.status.idle":"2022-05-22T11:42:25.702459Z","shell.execute_reply.started":"2022-05-22T11:42:25.691804Z","shell.execute_reply":"2022-05-22T11:42:25.701832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this code is based on: https://www.kaggle.com/c/histopathologic-cancer-detection/discussion/84132\n# WSI: Whole Slide Image - is a big image scan from the scanning of conventional glass slides\n# every WSI later is chopped into the samples that made this histological dataset\n# objective is to identify images from the same WSI, so you can separate clearly training \n# dataset from the validation dataset, otherwise training and validating on the same WSI \n# may provide too optimistic results, that will not work in the reality\n# Note: Add extra dataset with WSI IDs: 'tywangty/histopathologiccancerwsi'\n\ndef return_tumor_or_not(dic, one_id):\n    return dic[one_id]\n\ndef create_dict():\n    df = pd.read_csv(annotation_file)\n    result_dict = {}\n    for index in range(df.shape[0]):\n        one_id = df.iloc[index,0]\n        tumor_or_not = df.iloc[index,1]\n        result_dict[one_id] = int(tumor_or_not)\n    return result_dict\n\ndef find_missing(train_ids, cv_ids):\n    all_ids = set(pd.read_csv(annotation_file)['id'].values)\n    wsi_ids = set(train_ids + cv_ids)\n\n    missing_ids = list(all_ids-wsi_ids)\n    return missing_ids\n\n\ndef generate_split():\n    #ids = pd.read_csv(\"../input/histopathologiccancerwsi/patch_id_wsi.csv\")\n    ids = pd.read_csv(\"../input/histopathologicwsifull/patch_id_wsi_full.csv\")\n    wsi_dict = {}\n    for i in range(ids.shape[0]):\n        wsi = ids.iloc[i,1]\n        train_id = ids.iloc[i,0]\n        wsi_array = wsi.split('_')\n        number = int(wsi_array[3])\n        if wsi_dict.get(number) is None:\n            wsi_dict[number] = [train_id]\n        else:\n            wsi_dict[number].append(train_id)\n\n    wsi_keys = list(wsi_dict.keys())\n    # np.random.seed()\n    np.random.shuffle(wsi_keys)\n    amount_of_keys = len(wsi_keys)\n\n    keys_for_train = wsi_keys[0:int(amount_of_keys*0.8)]\n    keys_for_cv = wsi_keys[int(amount_of_keys*0.8):]\n    train_ids = []\n    cv_ids = []\n\n    for key in keys_for_train:\n        train_ids += wsi_dict[key]\n\n    for key in keys_for_cv:\n        cv_ids += wsi_dict[key]\n\n    dic = create_dict()\n\n    missing_ids = find_missing(train_ids, cv_ids)\n    missing_ids_total = len(missing_ids)\n    # np.random.seed()\n    np.random.shuffle(missing_ids)\n\n    train_missing_ids = missing_ids[0:int(missing_ids_total*0.8)]\n    cv_missing_ids = missing_ids[int(missing_ids_total*0.8):]\n\n    train_ids += train_missing_ids\n    cv_ids += cv_missing_ids\n\n    train_labels = []\n    cv_labels = []\n\n    train_tumor = 0\n    for one_id in train_ids:\n        temp = return_tumor_or_not(dic, one_id)\n        train_tumor += temp\n        train_labels.append(temp)\n\n    cv_tumor = 0\n    for one_id in cv_ids:\n        temp = return_tumor_or_not(dic, one_id)\n        cv_tumor += temp\n        cv_labels.append(temp)\n    total = len(train_ids) + len(cv_ids)\n\n    print(\"Amount of train labels: {}, {}/{}\".format(len(train_ids), train_tumor, len(train_ids)-train_tumor))\n    print(\"Amount of cv labels: {}, {}/{}\".format(len(cv_ids), cv_tumor, len(cv_ids) - cv_tumor))\n    print(\"Percentage of cv labels: {}\".format(len(cv_ids)/total))\n    print(\"Total labels: {}\".format(total))\n\n    return train_ids, cv_ids, train_labels, cv_labels\n\ntrain_ids, cv_ids, train_labels, cv_labels = generate_split()\n\n# end of code is from: https://www.kaggle.com/c/histopathologic-cancer-detection/discussion/84132","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:42:25.704641Z","iopub.execute_input":"2022-05-22T11:42:25.704881Z","iopub.status.idle":"2022-05-22T11:42:56.685211Z","shell.execute_reply.started":"2022-05-22T11:42:25.704853Z","shell.execute_reply":"2022-05-22T11:42:56.684563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove duplicates from train and validation sets based of the file duplicates.csv\n# there are around 390 duplicates, which is not really many compare to the total data\n# however I just to to clean data and this is one of the cleaning steps\n# Note: add dataset wsiduplicates\n# see this discussion about duplicates: https://www.kaggle.com/competitions/histopathologic-cancer-detection/discussion/84964\n\n# make pandas datasets for train and validation\ntrain_data = pd.DataFrame( { 'id':train_ids, 'label':train_labels }, columns = ['id', 'label'] ).set_index('id')\n#print(\"Train data shape\", train_data.shape)\nlent = len(train_data)\n\nval_data = pd.DataFrame( { 'id':cv_ids, 'label':cv_labels }, columns = ['id', 'label'] ).set_index('id')\n#print(\"Validation data shape\", val_data.shape)\nlenv = len(val_data)\n\n# read duplicates.csv\nduplicates = pd.read_csv(\"../input/wsiduplicates/duplicates.csv\")\nprint(\"Duplicates data shape\", duplicates.shape)\n\n# there is also 6 white and 1 black slides, karked as \"non-cancer\"\n# empty slides were discovered and listed here: https://www.kaggle.com/code/qitvision/a-complete-ml-pipeline-fast-ai\n# I want to keep 1 black and 1 white slide in training as non-cancer\n# I think any empty (black or white) crop should be classified by model as non-pathological\n# to remove extra white slides, add them into duplicates\nwhite_slides = [ '9071b424ec2e84deeb59b54d2450a6d0172cf701',\n    'c448cd6574108cf14514ad5bc27c0b2c97fc1a83', '54df3640d17119486e5c5f98019d2a92736feabc',\n    '5f30d325d895d873d3e72a82ffc0101c45cba4a8', '5a268c0241b8510465cb002c4452d63fec71028a' ]\nwdf = pd.DataFrame(white_slides, columns=['id1'])\nduplicates.drop( columns=['id2'], inplace=True)\nduplicates = duplicates.append(wdf)\nprint(\"Duplicates data shape with whites\", duplicates.shape)\n\nprint(\"Removing duplicates...\")\n# remove duplicates\ndroplist = duplicates.id1\nfor di in duplicates.id1:\n    if di in train_data.index:\n        train_data.drop( di, inplace=True )\n    if di in val_data.index:\n        val_data.drop( di, inplace=True )\n        \nprint(\"Train data duplicates removed\", lent - len(train_data))\nprint(\"Validation data duplicates removed\", lenv - len(val_data))","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:42:56.686312Z","iopub.execute_input":"2022-05-22T11:42:56.686713Z","iopub.status.idle":"2022-05-22T11:43:32.486426Z","shell.execute_reply.started":"2022-05-22T11:42:56.686675Z","shell.execute_reply":"2022-05-22T11:43:32.485495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# there is one black and one white slide, I want them to be in the training set, those are 'not cancer'\nbid = '9369c7278ec8bcc6c880d99194de09fc2bd4efbe' # one black slide\nwid = 'f6f1d771d14f7129a6c3ac2c220d90992c30c10b' # one white slide\nif bid in train_data.index:\n    print('Black is in training')\nif bid in val_data.index:\n    print('! Black is in validation')\n    bd = val_data.loc[bid]\n    train_data = train_data.append(bd)\n    val_data.drop( bid, inplace=True )\n    print('Black moved to training')\n    \nif wid in train_data.index:\n    print('White is in training')\nif wid in val_data.index:\n    print('! White is in validation')\n    wd = val_data.loc[wid]\n    train_data = train_data.append(wd)\n    val_data.drop( wid, inplace=True )\n    print('White moved to training')","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:43:32.487720Z","iopub.execute_input":"2022-05-22T11:43:32.488238Z","iopub.status.idle":"2022-05-22T11:43:32.539660Z","shell.execute_reply.started":"2022-05-22T11:43:32.488201Z","shell.execute_reply":"2022-05-22T11:43:32.538400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optionally reduce size of training and validation\n# this may help to dedug pre-run\ndef reduceDataset(td, reducedSize):\n    df_negative_class = td[td['label'] == 0]\n    df_positive_class = td[td['label'] == 1]\n    df_pos = resample(df_positive_class, replace=False, n_samples=reducedSize, random_state=10)\n    df_neg = resample(df_negative_class, replace=False, n_samples=reducedSize, random_state=10)\n    return pd.concat([df_pos, df_neg])\n\nif(REDUCE):\n    train_data = reduceDataset(train_data, reducedTrainingSize)\n    val_data = reduceDataset(val_data, reducedValidationSize)\n    print(val_data.shape)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:43:32.541448Z","iopub.execute_input":"2022-05-22T11:43:32.541806Z","iopub.status.idle":"2022-05-22T11:43:32.558781Z","shell.execute_reply.started":"2022-05-22T11:43:32.541762Z","shell.execute_reply":"2022-05-22T11:43:32.557913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot a piechart of 4 slices\ntdc = train_data.label.value_counts().tolist()\nvdc = val_data.label.value_counts().tolist()\nvc = tdc + vdc\nvcs = sum(vc)\nprint(tdc, vdc, vc)\nplt.pie(vc, labels=['Train:No Cancer', 'Train:Cancer', 'Val:No Cancer', 'Val:Cancer'], \n        autopct=lambda x: '{:.0f}'.format(x*vcs/100), startangle=0)\nplt.title(\"Training/validation data balance\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:43:32.560696Z","iopub.execute_input":"2022-05-22T11:43:32.561342Z","iopub.status.idle":"2022-05-22T11:43:32.730160Z","shell.execute_reply.started":"2022-05-22T11:43:32.561294Z","shell.execute_reply":"2022-05-22T11:43:32.729006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now I want to arrange training and validation in a single dataFrame for the **fastai** ImageDataLoaders.from_df loader. \nTo do so, I need to concatenate train and validation and add a column **isValidation** to mark validation samples.\nI don't use random split, because above was the attampt to make more \"rational\" sptit.\n","metadata":{}},{"cell_type":"code","source":"# arrange training and validation into a single dataFrame\n# shuffle both sets before concatenating, later it may help\n# for example, fastai will feed in validation data without shuffling\nfull_data = pd.concat([train_data, val_data])\nfull_data['isValidation'] = 0\n# mark validation data\nfor di in val_data.index:\n    full_data.loc[di]['isValidation'] = 1\nfull_data = shuffle(full_data)\nfull_data.to_csv(\"WSI_splitted_data.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:43:32.731888Z","iopub.execute_input":"2022-05-22T11:43:32.732997Z","iopub.status.idle":"2022-05-22T11:43:37.400077Z","shell.execute_reply.started":"2022-05-22T11:43:32.732932Z","shell.execute_reply":"2022-05-22T11:43:37.399194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print( full_data['isValidation'].value_counts() )\nprint( full_data.head() )","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:43:37.401147Z","iopub.execute_input":"2022-05-22T11:43:37.402137Z","iopub.status.idle":"2022-05-22T11:43:37.414380Z","shell.execute_reply.started":"2022-05-22T11:43:37.402075Z","shell.execute_reply":"2022-05-22T11:43:37.413313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data.reset_index()\nval_data = val_data.reset_index()\nfull_data = full_data.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:43:37.416091Z","iopub.execute_input":"2022-05-22T11:43:37.416702Z","iopub.status.idle":"2022-05-22T11:43:37.452866Z","shell.execute_reply.started":"2022-05-22T11:43:37.416653Z","shell.execute_reply":"2022-05-22T11:43:37.452080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example: fastai ImageDataLoaders","metadata":{}},{"cell_type":"code","source":"from fastai.vision.all import *  # note: .all is important","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:43:37.454332Z","iopub.execute_input":"2022-05-22T11:43:37.454714Z","iopub.status.idle":"2022-05-22T11:43:38.205025Z","shell.execute_reply.started":"2022-05-22T11:43:37.454668Z","shell.execute_reply":"2022-05-22T11:43:38.204027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below we can try fastai loader **ImageDataLoaders.from_df** \n\nIn principle, it is possible to use ImageDataLoaders.from_csv, but fasai requires CSV file to be in the same folder as training data and this may by tricky. Using regular pd.read_csv() offers better flexibility.","metadata":{}},{"cell_type":"code","source":"fresh_data = pd.read_csv('WSI_splitted_data.csv')\ndls = ImageDataLoaders.from_df(fresh_data, path=path, suff='.tif', valid_col='isValidation')","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:43:38.207837Z","iopub.execute_input":"2022-05-22T11:43:38.208115Z","iopub.status.idle":"2022-05-22T11:44:01.763808Z","shell.execute_reply.started":"2022-05-22T11:43:38.208066Z","shell.execute_reply":"2022-05-22T11:44:01.762683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.rng.seed(Seed) # seed random to get the same results every time\n# make sure our data loader is working, try to show it loading\nprint('Training samples')\ndls.train.show_batch(max_n=15)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:44:01.765012Z","iopub.execute_input":"2022-05-22T11:44:01.765245Z","iopub.status.idle":"2022-05-22T11:44:03.187783Z","shell.execute_reply.started":"2022-05-22T11:44:01.765217Z","shell.execute_reply":"2022-05-22T11:44:03.186598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Validation samples')\ndls.valid.show_batch(max_n=15)\n# validation samples should be shown in both classes 0 and 1\n# data should be shuffled in advance to achieve this","metadata":{"execution":{"iopub.status.busy":"2022-05-22T11:44:03.189263Z","iopub.execute_input":"2022-05-22T11:44:03.189577Z","iopub.status.idle":"2022-05-22T11:44:04.489313Z","shell.execute_reply.started":"2022-05-22T11:44:03.189539Z","shell.execute_reply":"2022-05-22T11:44:04.485592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Have a nice day!","metadata":{}}]}