{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport cv2\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport random\nfrom sklearn.utils import shuffle\nfrom tqdm import tqdm_notebook\n\ndata = pd.read_csv('/kaggle/input/histopathologic-cancer-detection/train_labels.csv')\ntrain_path = '/kaggle/input/histopathologic-cancer-detection/train/'\ntest_path = '/kaggle/input/histopathologic-cancer-detection/test/'\n# quick look at the label stats\ndata['label'].value_counts()\n\n#there are 1.5 times more negative images than positives","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-10T15:48:27.605062Z","iopub.execute_input":"2021-11-10T15:48:27.605598Z","iopub.status.idle":"2021-11-10T15:48:29.292911Z","shell.execute_reply.started":"2021-11-10T15:48:27.605496Z","shell.execute_reply":"2021-11-10T15:48:29.292341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['path'] = data.id + '.tif'","metadata":{"execution":{"iopub.status.busy":"2021-11-10T15:49:17.971667Z","iopub.execute_input":"2021-11-10T15:49:17.972117Z","iopub.status.idle":"2021-11-10T15:49:18.011986Z","shell.execute_reply.started":"2021-11-10T15:49:17.972088Z","shell.execute_reply":"2021-11-10T15:49:18.011287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-10T15:49:20.775251Z","iopub.execute_input":"2021-11-10T15:49:20.776002Z","iopub.status.idle":"2021-11-10T15:49:20.789477Z","shell.execute_reply.started":"2021-11-10T15:49:20.775967Z","shell.execute_reply":"2021-11-10T15:49:20.788447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(data.label.value_counts() / len(data)).to_frame().sort_index().T","metadata":{"execution":{"iopub.status.busy":"2021-11-10T15:49:26.566553Z","iopub.execute_input":"2021-11-10T15:49:26.567457Z","iopub.status.idle":"2021-11-10T15:49:26.585632Z","shell.execute_reply.started":"2021-11-10T15:49:26.567409Z","shell.execute_reply":"2021-11-10T15:49:26.584743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plot Images\ndef readImage(path):\n    # OpenCV reads the image in bgr format by default\n    bgr_img = cv2.imread(path)\n    # We flip it to rgb for visualization purposes\n    b,g,r = cv2.split(bgr_img)\n    rgb_img = cv2.merge([r,g,b])\n    return rgb_img","metadata":{"execution":{"iopub.status.busy":"2021-11-10T15:49:49.001182Z","iopub.execute_input":"2021-11-10T15:49:49.001487Z","iopub.status.idle":"2021-11-10T15:49:49.006915Z","shell.execute_reply.started":"2021-11-10T15:49:49.001458Z","shell.execute_reply":"2021-11-10T15:49:49.006026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# random sampling\nshuffled_data = shuffle(data)\n\nfig, ax = plt.subplots(2,5, figsize=(20,8))\nfig.suptitle('Histopathologic scans of lymph node sections',fontsize=20)\n# Negatives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 0]['id'][:5]):\n    path = os.path.join(train_path, idx)\n    ax[0,i].imshow(readImage(path + '.tif'))\n    # Create a Rectangle patch\n    box = patches.Rectangle((32,32),32,32,linewidth=4,edgecolor='b',facecolor='none', linestyle=':', capstyle='round')\n    ax[0,i].add_patch(box)\nax[0,0].set_ylabel('Negative samples', size='large')\n# Positives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 1]['id'][:5]):\n    path = os.path.join(train_path, idx)\n    ax[1,i].imshow(readImage(path + '.tif'))\n    # Create a Rectangle patch\n    box = patches.Rectangle((32,32),32,32,linewidth=4,edgecolor='r',facecolor='none', linestyle=':', capstyle='round')\n    ax[1,i].add_patch(box)\nax[1,0].set_ylabel('Tumor tissue samples', size='large')","metadata":{"execution":{"iopub.status.busy":"2021-11-10T15:49:58.377759Z","iopub.execute_input":"2021-11-10T15:49:58.378016Z","iopub.status.idle":"2021-11-10T15:50:00.422090Z","shell.execute_reply.started":"2021-11-10T15:49:58.377989Z","shell.execute_reply":"2021-11-10T15:50:00.420995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nORIGINAL_SIZE = 96      # original size of the images - do not change\n\n# AUGMENTATION VARIABLES\nCROP_SIZE = 90          # final size after crop\nRANDOM_ROTATION = 3    # range (0-180), 180 allows all rotation variations, 0=no change\nRANDOM_SHIFT = 2        # center crop shift in x and y axes, 0=no change. This cannot be more than (ORIGINAL_SIZE - CROP_SIZE)//2 \nRANDOM_BRIGHTNESS = 7  # range (0-100), 0=no change\nRANDOM_CONTRAST = 5    # range (0-100), 0=no change\nRANDOM_90_DEG_TURN = 1  # 0 or 1= random turn to left or right\n\ndef readCroppedImage(path, augmentations = True):\n    # augmentations parameter is included for counting statistics from images, where we don't want augmentations\n    \n    # OpenCV reads the image in bgr format by default\n    bgr_img = cv2.imread(path)\n    # We flip it to rgb for visualization purposes\n    b,g,r = cv2.split(bgr_img)\n    rgb_img = cv2.merge([r,g,b])\n    \n    if(not augmentations):\n        return rgb_img / 255\n    \n    #random rotation\n    rotation = random.randint(-RANDOM_ROTATION,RANDOM_ROTATION)\n    if(RANDOM_90_DEG_TURN == 1):\n        rotation += random.randint(-1,1) * 90\n    M = cv2.getRotationMatrix2D((48,48),rotation,1)   # the center point is the rotation anchor\n    rgb_img = cv2.warpAffine(rgb_img,M,(96,96))\n    \n    #random x,y-shift\n    x = random.randint(-RANDOM_SHIFT, RANDOM_SHIFT)\n    y = random.randint(-RANDOM_SHIFT, RANDOM_SHIFT)\n    \n    # crop to center and normalize to 0-1 range\n    start_crop = (ORIGINAL_SIZE - CROP_SIZE) // 2\n    end_crop = start_crop + CROP_SIZE\n    rgb_img = rgb_img[(start_crop + x):(end_crop + x), (start_crop + y):(end_crop + y)] / 255\n    \n    # Random flip\n    flip_hor = bool(random.getrandbits(1))\n    flip_ver = bool(random.getrandbits(1))\n    if(flip_hor):\n        rgb_img = rgb_img[:, ::-1]\n    if(flip_ver):\n        rgb_img = rgb_img[::-1, :]\n        \n    # Random brightness\n    br = random.randint(-RANDOM_BRIGHTNESS, RANDOM_BRIGHTNESS) / 100.\n    rgb_img = rgb_img + br\n    \n    # Random contrast\n    cr = 1.0 + random.randint(-RANDOM_CONTRAST, RANDOM_CONTRAST) / 100.\n    rgb_img = rgb_img * cr\n    \n    # clip values to 0-1 range\n    rgb_img = np.clip(rgb_img, 0, 1.0)\n    \n    return rgb_img","metadata":{"execution":{"iopub.status.busy":"2021-11-10T15:50:35.195464Z","iopub.execute_input":"2021-11-10T15:50:35.195775Z","iopub.status.idle":"2021-11-10T15:50:35.210042Z","shell.execute_reply.started":"2021-11-10T15:50:35.195742Z","shell.execute_reply":"2021-11-10T15:50:35.209250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(2,5, figsize=(20,8))\nfig.suptitle('Cropped histopathologic scans of lymph node sections',fontsize=20)\n# Negatives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 0]['id'][:5]):\n    path = os.path.join(train_path, idx)\n    ax[0,i].imshow(readCroppedImage(path + '.tif'))\nax[0,0].set_ylabel('Negative samples', size='large')\n# Positives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 1]['id'][:5]):\n    path = os.path.join(train_path, idx)\n    ax[1,i].imshow(readCroppedImage(path + '.tif'))\nax[1,0].set_ylabel('Tumor tissue samples', size='large')","metadata":{"execution":{"iopub.status.busy":"2021-11-10T15:50:43.330407Z","iopub.execute_input":"2021-11-10T15:50:43.331107Z","iopub.status.idle":"2021-11-10T15:50:45.192809Z","shell.execute_reply.started":"2021-11-10T15:50:43.331056Z","shell.execute_reply":"2021-11-10T15:50:45.191829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plot one image multiple times\nfig, ax = plt.subplots(1,5, figsize=(20,4))\nfig.suptitle('Random augmentations to the same image',fontsize=20)\n# Negatives\nfor i, idx in enumerate(shuffled_data[shuffled_data['label'] == 0]['id'][:1]):\n    for j in range(5):\n        path = os.path.join(train_path, idx)\n        ax[j].imshow(readCroppedImage(path + '.tif'))","metadata":{"execution":{"iopub.status.busy":"2021-11-10T15:50:47.985903Z","iopub.execute_input":"2021-11-10T15:50:47.986166Z","iopub.status.idle":"2021-11-10T15:50:49.097047Z","shell.execute_reply.started":"2021-11-10T15:50:47.986137Z","shell.execute_reply":"2021-11-10T15:50:49.096096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# As we count the statistics, we can check if there are any completely black or white images\ndark_th = 10 / 255      # If no pixel reaches this threshold, image is considered too dark\nbright_th = 245 / 255   # If no pixel is under this threshold, image is considerd too bright\ntoo_dark_idx = []\ntoo_bright_idx = []\n\nx_tot = np.zeros(3)\nx2_tot = np.zeros(3)\ncounted_ones = 0\nfor i, idx in tqdm_notebook(enumerate(shuffled_data['id']), 'computing statistics...(220025 it total)'):\n    path = os.path.join(train_path, idx)\n    imagearray = readCroppedImage(path + '.tif', augmentations = False).reshape(-1,3)\n    # is this too dark\n    if(imagearray.max() < dark_th):\n        too_dark_idx.append(idx)\n        continue # do not include in statistics\n    # is this too bright\n    if(imagearray.min() > bright_th):\n        too_bright_idx.append(idx)\n        continue # do not include in statistics\n    x_tot += imagearray.mean(axis=0)\n    x2_tot += (imagearray**2).mean(axis=0)\n    counted_ones += 1\n    \nchannel_avr = x_tot/counted_ones\nchannel_std = np.sqrt(x2_tot/counted_ones - channel_avr**2)\nchannel_avr,channel_std","metadata":{"execution":{"iopub.status.busy":"2021-11-10T15:50:50.679267Z","iopub.execute_input":"2021-11-10T15:50:50.679584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('There was {0} extremely dark image'.format(len(too_dark_idx)))\nprint('and {0} extremely bright images'.format(len(too_bright_idx)))\nprint('Dark one:')\nprint(too_dark_idx)\nprint('Bright ones:')\nprint(too_bright_idx)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# take a smaller sample to run the model\nsample_data, ignore = train_test_split(data, test_size=0.99, random_state=1, stratify=data.label)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample_data.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain, valid = train_test_split(sample_data, test_size=0.2, random_state=1, stratify=sample_data.label)\n\nprint(train.shape)\nprint(valid.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# we read the csv file earlier to pandas dataframe, now we set index to id so we can perform\ntrain_df = data.set_index('id')\n\n#If removing outliers, uncomment the four lines below\n#print('Before removing outliers we had {0} training samples.'.format(train_df.shape[0]))\n#train_df = train_df.drop(labels=too_dark_idx, axis=0)\n#train_df = train_df.drop(labels=too_bright_idx, axis=0)\n#print('After removing outliers we have {0} training samples.'.format(train_df.shape[0]))\n\ntrain_names = train_df.index.values\ntrain_labels = np.asarray(train_df['label'].values)\n\n# split, this function returns more than we need as we only need the validation indexes for fastai\ntr_n, tr_idx, val_n, val_idx = train_test_split(train_names, range(len(train_names)), test_size=0.1, stratify=train_labels, random_state=123)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}