{"cells":[{"metadata":{},"cell_type":"markdown","source":"**References**\nBelow are few of the references that helped me understand and participate in this competition.\n- https://www.kaggle.com/dschettler8845/hpa-cellwise-classification-inference/data\n- https://www.kaggle.com/its7171/hpa-mask\n- https://www.kaggle.com/thedrcat/hpa-single-cell-classification-eda\n- https://towardsdatascience.com/multi-label-image-classification-in-tensorflow-2-0-7d4cf8a4bc72\n- Sechidis, K., Tsoumakas, G., & Vlahavas, I. (2011). On the stratification of multi-label data. Machine Learning and Knowledge Discovery in Databases, 145-158\n- And many more!!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\nfrom pathlib import Path\nimport imageio\nfrom time import time\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom keras.preprocessing import image\nfrom tensorflow.keras import layers\nimport matplotlib.style as style\nfrom keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_SIZE=256\nN_LABELS = 19\nEPOCHS = 1\nTF_BATCH_SIZE = 256 # Big enough to measure an F1-score\nAUTOTUNE = tf.data.experimental.AUTOTUNE # Adapt preprocessing and prefetching dynamically to reduce GPU and CPU idle time\nSHUFFLE_BUFFER_SIZE = 1024 # Shuffle the training data by a chunck of 1024 observations\n\nIMG_SIZE = 256 # Specify height and width of image to match the input format of the model\nCHANNELS = 3 # Keep RGB color channels to match the input format of the model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_function(filename, label):\n    \"\"\"Function that returns a tuple of normalized image array and labels array.\n    Args:\n        filename: string representing path to image\n        label: 0/1 one-dimensional array of size N_LABELS\n    \"\"\"\n    # Read an image from a file\n    image_string = tf.io.read_file(filename)\n    # Decode it into a dense vector\n    image_decoded = tf.image.decode_jpeg(image_string, channels=CHANNELS)\n    # Resize it to fixed shape\n    image_resized = tf.image.resize(image_decoded, [IMG_SIZE, IMG_SIZE])\n    # Normalize it from [0, 255] to [0.0, 1.0]\n    image_normalized = image_resized / 255.0\n#     print(image_normalized)\n    return image_normalized, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_dataset(filenames, labels, is_training=True):\n    \"\"\"Load and parse dataset.\n    Args:\n        filenames: list of image paths\n        labels: numpy array of shape (TF_BATCH_SIZE, N_LABELS)\n        is_training: boolean to indicate training mode\n    \"\"\"\n    \n    # Create a first dataset of file paths and labels\n    dataset = tf.data.Dataset.from_tensor_slices((filenames, labels))\n    # Parse and preprocess observations in parallel\n    dataset = dataset.map(parse_function, num_parallel_calls=AUTOTUNE)\n    \n#     if is_training == True:\n#         # This is a small dataset, only load it once, and keep it in memory.\n#         dataset = dataset.cache()\n#         # Shuffle the data each buffer size\n#         dataset = dataset.shuffle(buffer_size=SHUFFLE_BUFFER_SIZE)\n        \n    # Batch the data for multiple steps\n    dataset = dataset.batch(TF_BATCH_SIZE)\n    # Fetch batches in the background while the model is training.\n    dataset = dataset.prefetch(buffer_size=AUTOTUNE)\n    \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef macro_soft_f1(y, y_hat):\n    \"\"\"Compute the macro soft F1-score as a cost (average 1 - soft-F1 across all labels).\n    Use probability values instead of binary predictions.\n    \n    Args:\n        y (int32 Tensor): targets array of shape (TF_BATCH_SIZE, N_LABELS)\n        y_hat (float32 Tensor): probability matrix from forward propagation of shape (TF_BATCH_SIZE, N_LABELS)\n        \n    Returns:\n        cost (scalar Tensor): value of the cost function for the batch\n    \"\"\"\n    y = tf.cast(y, tf.float32)\n    y_hat = tf.cast(y_hat, tf.float32)\n    tp = tf.reduce_sum(y_hat * y, axis=0)\n    fp = tf.reduce_sum(y_hat * (1 - y), axis=0)\n    fn = tf.reduce_sum((1 - y_hat) * y, axis=0)\n    soft_f1 = 2*tp / (2*tp + fn + fp + 1e-16)\n    cost = 1 - soft_f1 # reduce 1 - soft-f1 in order to increase soft-f1\n    macro_cost = tf.reduce_mean(cost) # average on all labels\n    return macro_cost","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"physical_devices = tf.config.list_physical_devices('GPU')\ntry:\n  tf.config.experimental.set_memory_growth(physical_devices[0], True)\nexcept:\n  # Invalid device or cannot modify virtual devices once initialized.\n  pass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install \"../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"\n!pip install \"../input/hpapytorchzoozip/pytorch_zoo-master\"\n!pip install \"../input/hpacellsegmentatorraman/HPA-Cell-Segmentation/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pycocotools import _mask as coco_mask\nimport typing as t\nimport base64\nimport zlib\n\ndef binary_mask_to_ascii(mask, mask_val=1):\n    \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n    mask = np.where(mask==mask_val, 1, 0).astype(np.bool)\n    \n    # check input mask --\n    if mask.dtype != np.bool:\n        raise ValueError(f\"encode_binary_mask expects a binary mask, received dtype == {mask.dtype}\")\n\n    mask = np.squeeze(mask)\n    if len(mask.shape) != 2:\n        raise ValueError(f\"encode_binary_mask expects a 2d mask, received shape == {mask.shape}\")\n\n    # convert input mask to expected COCO API input --\n    mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    mask_to_encode = mask_to_encode.astype(np.uint8)\n    mask_to_encode = np.asfortranarray(mask_to_encode)\n\n    # RLE encode mask --\n    encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n    # compress and base64 encoding --\n    binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n    base64_str = base64.b64encode(binary_str)\n    return base64_str.decode()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nif not os.path.exists('cells-segmented'):\n    os.makedirs('cells-segmented')\nif not os.path.exists('cells-segmented/test'):\n    os.makedirs('cells-segmented/test')\nif not os.path.exists('test-masks'):\n    os.makedirs('test-masks')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import glob\n# red_test_globs = glob.glob('../input/hpa-single-cell-image-classification/test/*_red.png')\n# test_imageids = []\n# for name in red_test_globs:\n#     tokens = name.split('/')\n#     image_id = tokens[len(tokens) - 1].split('_red')[0]\n#     test_imageids.append({'ID': image_id})\n# test_df = pd.DataFrame(test_imageids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/hpa-single-cell-image-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_IMAGE_SIZE = 1024\nBATCH_SIZE = 24","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_cell(img, mask):\n    mask = mask.reshape(mask.shape[0], mask.shape[1], 1)\n    img_mask = img * mask\n    non_zero_points = np.argwhere(img_mask[:,:,:])\n    max_xy = non_zero_points.max(axis=0)\n    min_xy = non_zero_points.min(axis=0)\n    return img_mask[min_xy[0]:max_xy[0] + 1,min_xy[1]:max_xy[1] + 1,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_img(image_id, color, train_or_test='test', image_size=None):\n    filename = f'../input/hpa-single-cell-image-classification/{train_or_test}/{image_id}_{color}.png'\n    assert os.path.exists(filename), f'not found {filename}'\n    img = cv2.imread(filename, cv2.IMREAD_GRAYSCALE)\n    if image_size is not None:\n        img = cv2.resize(img, (image_size, image_size))\n    if img.max() > 255:\n        img_max = img.max()\n        img = (img/255).astype('uint8')\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_images(df, train_or_test = 'test'):\n    image_path_prefix = f'../input/hpa-single-cell-image-classification/{train_or_test}/'\n    \n    red_images = [cv2.imread(f'{image_path_prefix}/{row.ID}_red.png', cv2.IMREAD_GRAYSCALE) for _,row in df.iterrows()]\n    green_images = [cv2.imread(f'{image_path_prefix}/{row.ID}_green.png', cv2.IMREAD_GRAYSCALE) for _,row in df.iterrows()]\n    blue_images = [cv2.imread(f'{image_path_prefix}/{row.ID}_blue.png', cv2.IMREAD_GRAYSCALE) for _,row in df.iterrows()]\n    # 24x512x512\n    height_widths = [red_images[i].shape for i in range(len(red_images))]\n    blue_image_scaled = [cv2.resize(b, (TEST_IMAGE_SIZE, TEST_IMAGE_SIZE)) / 255. for b in blue_images]\n    rgb_image_scaled = [cv2.resize(np.stack((red_images[i], green_images[i], blue_images[i]), axis=2), (TEST_IMAGE_SIZE, TEST_IMAGE_SIZE)) / 255.\n                        for i in range(len(red_images))]\n    # 24x3x1024x1024\n    return blue_image_scaled, rgb_image_scaled, height_widths","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def segment_cells(df, train_or_test='test'):\n    all_cells = []\n    for index, row in df.iterrows():\n        image_id = row.ID\n#         if index % 50 == 0:\n#         print(f'Working on ImageId={image_id}, index={index}')\n        if train_or_test == 'train':\n            cell_mask = np.load(f'{DATASET_HPA_MASK}/hpa_cell_mask/{image_id}.npz')['arr_0']\n        else:\n            cell_mask = np.load(f'test-masks/{image_id}.npz')['arr_0']\n        red_image = read_img(image_id, color='red', train_or_test=train_or_test, image_size=TEST_IMAGE_SIZE)\n        green_image = read_img(image_id, color='green', train_or_test=train_or_test, image_size=TEST_IMAGE_SIZE)\n        blue_image = read_img(image_id, color='blue', train_or_test=train_or_test, image_size=TEST_IMAGE_SIZE)\n        img = np.dstack((blue_image, green_image, red_image))\n        for i in range(1, np.max(cell_mask) + 1):\n#             print(f'Working on cell={i}')\n            if train_or_test == 'train':\n                all_cells.append({\n                    'image_id': image_id,\n                    'cell_no': i,\n                    'labels': row.Label\n                })\n            else:\n                all_cells.append({\n                    'image_id': image_id,\n                    'cell_no': i,\n                })\n            bin_mask = cell_mask == i;\n            cell = crop_cell(img, bin_mask)\n            cv2.imwrite(f'./cells-segmented/{train_or_test}/{image_id}_{i}.jpg', cell)\n    return pd.DataFrame(all_cells)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_cell_masks(df, train_or_test='test'):\n    height_widths = []\n    for start in range(0, len(df), BATCH_SIZE):\n        end = min(start + BATCH_SIZE, len(df))\n        chunk = df[start:end]\n#         print(str(start) + '-' + str(end))\n        blue_images, images, hws = load_images(chunk, train_or_test=train_or_test)\n        height_widths += hws\n        # For nuclei\n        nuc_segmentations = segmentator.pred_nuclei(blue_images)\n        # For full cells\n        cell_segmentations = segmentator.pred_cells(images, precombined=True)\n        # post-processing\n        for i, pred in enumerate(cell_segmentations):\n            nuclei_mask, cell_mask = utils.label_cell(nuc_segmentations[i], cell_segmentations[i])\n            if train_or_test == 'test':\n                image_id = df.iloc[start + i].ID\n                np.savez_compressed(f'test-masks/{image_id}', cell_mask)\n    return height_widths","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from hpacellseg import cellsegmentator, utils\n    \nNUC_MODEL = \"../input/hpacellsegmentatormodelweights/dpn_unet_nuclei_v1.pth\"\nCELL_MODEL = \"../input/hpacellsegmentatormodelweights/dpn_unet_cell_3ch_v1.pth\"\nsegmentator = cellsegmentator.CellSegmentator(\n    NUC_MODEL,\n    CELL_MODEL,\n    device=\"cuda\",\n    multi_channel_model=True,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Starting cell segmentation')\ntest_height_widths = generate_cell_masks(test_df, train_or_test='test')\ntest_cells_df = segment_cells(test_df, train_or_test='test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.imshow(np.load(f'test-masks/020a29cf-2c24-478b-8603-c22a90dc3e31.npz')['arr_0'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from IPython.display import Image\n# # Image(f'../input/hpa-single-cell-image-classification/test/020a29cf-2c24-478b-8603-c22a90dc3e31_yellow.png')\n# Image(f'./cells-segmented/test/84895b21-d582-4fc8-b8a4-7ffcbab587a8_40.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\ntorch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_cells_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test_sub = ['./cells-segmented/test/' + str(row.image_id) + '_' + str(row.cell_no) + '.jpg' for _, row in test_cells_df.iterrows()]\nprint(X_test_sub[0:2])\nX_test_dict = {row.image_id + '_' + str(row.cell_no) : index for index, row in test_cells_df.iterrows()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nmodel = keras.models.load_model('../input/hpa-train-model/model-dl.h5', custom_objects={'macro_soft_f1':macro_soft_f1})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.weights[12]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_test_sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.imshow(parse_function('./cells-segmented/test/9b1d4b27-6946-4b86-a818-8e91029c3dfa_1.jpg', None)[0].numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = create_dataset(X_test_sub, X_test_sub, False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for z in test_ds:\n#   print(plt.imshow(z[0][10]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predictions = model.predict(test_ds, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"THRESHOLD = 0.05\nsubmission_data = []\nfor index, row in test_df.iterrows():\n    height_width = test_height_widths[index]\n    image_id = row.ID\n    mask = np.load(f'./test-masks/{image_id}.npz')['arr_0']\n    predictions = []\n    cell_id = 1\n    while f'{image_id}_{cell_id}' in X_test_dict:\n        cell_index = X_test_dict[f'{image_id}_{cell_id}']\n#         print(cell_index)\n        pred = test_predictions[cell_index]\n#         print(pred)\n#         filtered_preds = {i: pred[i] for i in range(N_LABELS)}\n        submission_rle = binary_mask_to_ascii(mask, mask_val=cell_id)\n        cell_predictions = [f'{i} {p} {submission_rle}' for i,p in enumerate(pred) if p >= THRESHOLD]\n        predictions += cell_predictions\n        cell_id += 1\n    submission_data.append({\n        'ID': image_id,\n        'ImageWidth': height_width[0],\n        'ImageHeight': height_width[1],\n        'PredictionString': ' '.join(predictions)\n    })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(submission_data).to_csv('./submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fig, ax = plt.subplots(5, 2, figsize=(20,50))\n\n# for i, data_id in enumerate(train_df.ID.to_list()[:5]):\n    \n#     cell_image = np.stack([\n#         cv2.imread(f'../input/hpa-single-cell-image-classification/train/{data_id}_red.png', 0),\n#         cv2.imread(f'../input/hpa-single-cell-image-classification/train/{data_id}_yellow.png', 0),\n#         cv2.imread(f'../input/hpa-single-cell-image-classification/train/{data_id}_blue.png', 0)], axis=2)\n#     cell_image = cv2.resize(cell_image, (512, 512))\n#     ax[i, 0].imshow(cell_image)\n#     ax[i, 0].imshow(train_cell_masks[i], alpha=0.5)\n#     ax[i, 0].axis('off')\n#     ax[i, 0].set_title('Faster')\n    \n#     ax[i, 1].imshow(train_cell_masks[i])\n#     ax[i, 1].axis('off')\n#     ax[i, 1].set_title('Just cell masks')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf test-masks\n!rm -rf cells-segmented","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}