{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Creating a prototyping dataset with individual cells\n\nMy full solution is described here: https://www.kaggle.com/c/hpa-single-cell-image-classification/discussion/221550\n\nWhat I need as an input to the classification model are images of individual cells. For experimentation I don't need all the images, instead I create a sample from the train set. The additional benefit is that my sample is more balanced than train. I use RGB channels only, which has proven to work well in the previous HPA challenge. I save the extracted cells as RGB jpg images so that I can feed them easily into my classifier.\n\nAcknowledgements - this uses the dataset and some code by @its7171 (please upvote!):\n- https://www.kaggle.com/its7171/hpa-mask\n- https://www.kaggle.com/its7171/mmdetection-for-segmentation-training/"},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision.all import *\nimport pandas as pd\nimport numpy as np\nimport os\nfrom tqdm import tqdm\nimport cv2\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**#FILTER**"},{"metadata":{"trusted":true},"cell_type":"code","source":"dfd = pd.read_csv('../input/hpa-duplicate-images-in-train/hpa_duplicates.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfd.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('../input/hpa-single-cell-image-classification')\ndf = pd.read_csv(path/'train.csv')\ncell_dir = '../input/hpa-mask/hpa_cell_mask'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(dfd)):\n    index = df[df['ID'] == dfd['image2'][i]].index\n    df= df.drop(index, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = '../input/hpa-single-cell-image-classification/'\ntrain_or_test = 'train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = [str(i) for i in range(19)]\nfor x in labels: df[x] = df['Label'].apply(lambda r: int(x in r.split('|')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs_0 = df[df['Label'] == '0'].sample(n=300, random_state=42).reset_index(drop=True)\ndfs_1 = df[df['1'] == 1].sample(n=400, random_state=42).reset_index(drop=True)\ndfs_1u = df[df['Label'] == '1'].sample(frac=1, random_state=42).reset_index(drop=True)\ndfs_2 = df[df['Label'] == '2'].sample(frac=1, random_state=42).reset_index(drop=True)\ndfs_3 = df[df['Label'] == '3'].sample(n=500, random_state=42).reset_index(drop=True)\ndfs_4 = df[df['Label'] == '4'].sample(n=500, random_state=42).reset_index(drop=True)\ndfs_5 = df[df['Label'] == '5'].sample(n=500, random_state=42).reset_index(drop=True)\ndfs_6 = df[df['6'] == 1].sample(n=600, random_state=42).reset_index(drop=True)\ndfs_7 = df[df['Label'] == '7'].sample(n=500, random_state=42).reset_index(drop=True)\ndfs_8 = df[df['Label'] == '8'].sample(n=500, random_state=42).reset_index(drop=True)\ndfs_9 = df[df['9'] == 1].sample(n=400, random_state=42).reset_index(drop=True)\ndfs_9u = df[df['Label'] == '9'].sample(n=200, random_state=42).reset_index(drop=True)\ndfs_10 = df[df['10'] == 1].sample(n=400, random_state=42).reset_index(drop=True)\ndfs_10u = df[df['Label'] == '10'].sample(n=200, random_state=42).reset_index(drop=True)\ndfs_11 = df[df['11'] == 1].sample(frac=1, random_state=42, replace= True).reset_index(drop=True).reset_index(drop=True)\ndfs_12 = df[df['Label'] == '12'].sample(frac=1, random_state=42).reset_index(drop=True)\ndfs_13 = df[df['Label'] == '13'].sample(n=400, random_state=42).reset_index(drop=True)\ndfs_14 = df[df['Label'] == '14'].sample(n=500, random_state=42).reset_index(drop=True)\ndfs_15 = df[df['15'] == 1].reset_index(drop=True)\ndfs_16 = df[df['Label'] == '16'].sample(n=350, random_state=42).reset_index(drop=True)\ndfs_17 = df[df['17'] == 1].sample(frac=1, random_state=42).reset_index(drop=True)\ndfs_18 = df[df['18'] == 1].sample(frac=1, random_state=42).reset_index(drop=True)\ndfs_ = [dfs_0, dfs_1, dfs_1u, dfs_2, dfs_3, dfs_4, dfs_5, dfs_6, dfs_7, dfs_8, dfs_9, dfs_9u, dfs_10, dfs_10u,\n        dfs_11, dfs_12, dfs_13, dfs_14, dfs_15, dfs_16, dfs_17, dfs_18]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs = pd.concat(dfs_, ignore_index=True)\ndfs.drop_duplicates(inplace=True, ignore_index=True)\nlen(dfs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_counts = {}\nfor lbl in labels:\n    unique_counts[lbl] = len(dfs[dfs.Label == lbl])\n\nfull_counts = {}\nfor lbl in labels:\n    count = 0\n    for row_label in dfs['Label']:\n        if lbl in row_label.split('|'): count += 1\n    full_counts[lbl] = count\n    \ncounts = list(zip(full_counts.keys(), full_counts.values(), unique_counts.values()))\ncounts = np.array(sorted(counts, key=lambda x:-x[1]))\ncounts = pd.DataFrame(counts, columns=['label', 'full_count', 'unique_count'])\ncounts.set_index('label').T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_cropped_cell(img, msk):\n    bmask = msk.astype(int)[...,None]\n    masked_img = img * bmask\n    true_points = np.argwhere(bmask)\n    top_left = true_points.min(axis=0)\n    bottom_right = true_points.max(axis=0)\n    cropped_arr = masked_img[top_left[0]:bottom_right[0]+1,top_left[1]:bottom_right[1]+1]\n    return cropped_arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_stats(cropped_cell):\n    x = (cropped_cell/255.0).reshape(-1,3).mean(0)\n    x2 = ((cropped_cell/255.0)**2).reshape(-1,3).mean(0)\n    return x, x2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_img(image_id, color, train_or_test='train', image_size=None):\n    filename = f'{ROOT}/{train_or_test}/{image_id}_{color}.png'\n    assert os.path.exists(filename), f'not found {filename}'\n    img = cv2.imread(filename, cv2.IMREAD_UNCHANGED)\n    if image_size is not None:\n        img = cv2.resize(img, (image_size, image_size))\n    if img.max() > 255:\n        img_max = img.max()\n        img = (img/255).astype('uint8')\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs.Label.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.mkdir('/kaggle/working/cells')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_tot,x2_tot = [],[]\nlbls = []\nnum_files = len(dfs)\nall_cells = []\ncell_mask_dir = '../input/hpa-mask/hpa_cell_mask'\n\nwith zipfile.ZipFile('cells.zip', 'w') as img_out:\n\n    for idx in tqdm(range(num_files)):\n        image_id = dfs.iloc[idx].ID\n        labels = dfs.iloc[idx].Label\n        cell_mask = np.load(f'{cell_mask_dir}/{image_id}.npz')['arr_0']\n        red = read_img(image_id, \"red\", train_or_test, None)\n        green = read_img(image_id, \"green\", train_or_test, None)\n        blue = read_img(image_id, \"blue\", train_or_test, None)\n        #yellow = read_img(image_id, \"yellow\", train_or_test, image_size)\n        stacked_image = np.transpose(np.array([blue, green, red]), (1,2,0))\n\n        for cell in range(1, np.max(cell_mask)):\n            bmask = cell_mask == cell\n            cropped_cell = get_cropped_cell(stacked_image, bmask)\n            fname = f'{image_id}_{cell}.jpg'\n            im = cv2.imencode('.jpg', cropped_cell)[1]\n            img_out.writestr(fname, im)\n            x, x2 = get_stats(cropped_cell)\n            x_tot.append(x)\n            x2_tot.append(x2)\n            all_cells.append({\n                'image_id': image_id,\n                'r_mean': x[0],\n                'g_mean': x[1],\n                'b_mean': x[2],\n                'cell_id': cell,\n                'image_labels': labels,\n                'size1': cropped_cell.shape[0],\n                'size2': cropped_cell.shape[1],\n            })\n\n#image stats\nimg_avr =  np.array(x_tot).mean(0)\nimg_std =  np.sqrt(np.array(x2_tot).mean(0) - img_avr**2)\ncell_df = pd.DataFrame(all_cells)\ncell_df.to_csv('cell_df.csv', index=False)\nprint('mean:',img_avr, ', std:', img_std)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cell_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -l --block-size=M","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cell_df.g_mean.hist(bins=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cell_df.r_mean.hist(bins=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cell_df.b_mean.hist(bins=100)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}