{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Creating a prototyping dataset with individual cells (test set)\n\nMy full solution is described here: https://www.kaggle.com/c/hpa-single-cell-image-classification/discussion/221550\n\nWhat I need as an input to the classification model are images of individual cells. For experimentation I don't need all the images, instead I create a sample from the train set. The additional benefit is that my sample is more balanced than train. I use RGB channels only, which has proven to work well in the previous HPA challenge. I save the extracted cells as RGB jpg images so that I can feed them easily into my classifier.\n\n## This is the notebook to create public test dataset processed in the same way as the training sample dataset.\n\n### Kind people upvote useful notebooks and datasets :) \n\n\nAcknowledgements - this uses the dataset and some code by @its7171 (please upvote!):\n- https://www.kaggle.com/its7171/hpa-mask\n- https://www.kaggle.com/its7171/mmdetection-for-segmentation-training/"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q \"../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision.all import *\nimport pandas as pd\nimport numpy as np\nimport os\nfrom tqdm import tqdm\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('../input/hpa-single-cell-image-classification')\ndf = pd.read_csv(path/'sample_submission.csv')\ncell_dir = '../input/hpa-cell-masks-test-dataset/work/cell_masks'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = '../input/hpa-single-cell-image-classification/'\ntrain_or_test = 'test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_cropped_cell(img, msk):\n    bmask = msk.astype(int)[...,None]\n    masked_img = img * bmask\n    true_points = np.argwhere(bmask)\n    top_left = true_points.min(axis=0)\n    bottom_right = true_points.max(axis=0)\n    cropped_arr = masked_img[top_left[0]:bottom_right[0]+1,top_left[1]:bottom_right[1]+1]\n    return cropped_arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_stats(cropped_cell):\n    x = (cropped_cell/255.0).reshape(-1,3).mean(0)\n    x2 = ((cropped_cell/255.0)**2).reshape(-1,3).mean(0)\n    return x, x2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_img(image_id, color, train_or_test='test', image_size=None):\n    filename = f'{ROOT}/{train_or_test}/{image_id}_{color}.png'\n    assert os.path.exists(filename), f'not found {filename}'\n    img = cv2.imread(filename, cv2.IMREAD_UNCHANGED)\n    if image_size is not None:\n        img = cv2.resize(img, (image_size, image_size))\n    if img.max() > 255:\n        img_max = img.max()\n        img = (img/255).astype('uint8')\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import base64\nimport numpy as np\nfrom pycocotools import _mask as coco_mask\nimport typing as t\nimport zlib\n\n\ndef encode_binary_mask(mask: np.ndarray) -> t.Text:\n  \"\"\"Converts a binary mask into OID challenge encoding ascii text.\"\"\"\n\n  # check input mask --\n  if mask.dtype != np.bool:\n    raise ValueError(\n        \"encode_binary_mask expects a binary mask, received dtype == %s\" %\n        mask.dtype)\n\n  mask = np.squeeze(mask)\n  if len(mask.shape) != 2:\n    raise ValueError(\n        \"encode_binary_mask expects a 2d mask, received shape == %s\" %\n        mask.shape)\n\n  # convert input mask to expected COCO API input --\n  mask_to_encode = mask.reshape(mask.shape[0], mask.shape[1], 1)\n  mask_to_encode = mask_to_encode.astype(np.uint8)\n  mask_to_encode = np.asfortranarray(mask_to_encode)\n\n  # RLE encode mask --\n  encoded_mask = coco_mask.encode(mask_to_encode)[0][\"counts\"]\n\n  # compress and base64 encoding --\n  binary_str = zlib.compress(encoded_mask, zlib.Z_BEST_COMPRESSION)\n  base64_str = base64.b64encode(binary_str)\n  return base64_str.decode('ascii')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_tot,x2_tot = [],[]\nlbls = []\nnum_files = len(df)\nall_cells = []\ncell_mask_dir = '../input/hpa-cell-masks-test-dataset/work/cell_masks'\n\nwith zipfile.ZipFile('cells.zip', 'w') as img_out:\n\n    for idx in tqdm(range(num_files)):\n        image_id = df.iloc[idx].ID\n        cell_mask = np.load(f'{cell_mask_dir}/{image_id}.npz')['arr_0']\n        red = read_img(image_id, \"red\", train_or_test, None)\n        green = read_img(image_id, \"green\", train_or_test, None)\n        blue = read_img(image_id, \"blue\", train_or_test, None)\n        #yellow = read_img(image_id, \"yellow\", train_or_test, image_size)\n        stacked_image = np.transpose(np.array([blue, green, red]), (1,2,0))\n\n        for j in range(1, np.max(cell_mask) + 1):\n            bmask = (cell_mask == j)\n            enc = encode_binary_mask(bmask)\n            cropped_cell = get_cropped_cell(stacked_image, bmask)\n            fname = f'{image_id}_{j}.jpg'\n            im = cv2.imencode('.jpg', cropped_cell)[1]\n            img_out.writestr(fname, im)\n            x, x2 = get_stats(cropped_cell)\n            x_tot.append(x)\n            x2_tot.append(x2)\n            all_cells.append({\n                'image_id': image_id,\n                'fname': fname,\n                'r_mean': x[0],\n                'g_mean': x[1],\n                'b_mean': x[2],\n                'cell_id': j,\n                'size1': cropped_cell.shape[0],\n                'size2': cropped_cell.shape[1],\n                'enc': enc,\n            })\n\n#image stats\nimg_avr =  np.array(x_tot).mean(0)\nimg_std =  np.sqrt(np.array(x2_tot).mean(0) - img_avr**2)\ncell_df = pd.DataFrame(all_cells)\ncell_df.to_csv('cell_df.csv', index=False)\nprint('mean:',img_avr, ', std:', img_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cell_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -l --block-size=M","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cell_df.g_mean.hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cell_df.r_mean.hist(bins=100);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cell_df.b_mean.hist(bins=100);","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}