{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Human Protein Atlas - Single Cell Classification

\n

Categorical Classification At a Cellular Level [TRAINING]

\n
CREATED BY: DARIEN SCHETTLER
\n\n

","metadata":{"papermill":{"duration":0.018998,"end_time":"2021-01-31T11:44:38.653319","exception":false,"start_time":"2021-01-31T11:44:38.634321","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

\n\n

\n\n

TABLE OF CONTENTS

\n\n
\n\n---\n\n

0    IMPORTS

\n\n---\n\n

1    BACKGROUND INFORMATION

\n\n---\n\n

2    SETUP

\n\n---\n\n

3    HELPER FUNCTIONS

\n\n---\n\n

4    PREPARING THE DATASET - TF.DATA

\n\n---\n\n

5    MODELLING

\n\n---","metadata":{"papermill":{"duration":0.017336,"end_time":"2021-01-31T11:44:38.688554","exception":false,"start_time":"2021-01-31T11:44:38.671218","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

0  IMPORTS    ⤒

","metadata":{"papermill":{"duration":0.0173,"end_time":"2021-01-31T11:44:38.723672","exception":false,"start_time":"2021-01-31T11:44:38.706372","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Try and get keras plot to work\n!pip install -q pydot\n!pip install -q pydotplus\n!apt-get install -q graphviz\n\nprint(\"\\n... OTHER IMPORTS STARTING ...\\n\")\nprint(\"\\n\\tVERSION INFORMATION\")\n\n# Machine Learning and Data Science Imports\nimport tensorflow_addons as tfa; print(f\"\\t\\t– TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport tensorflow as tf; print(f\"\\t\\t– TENSORFLOW VERSION: {tf.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\t– NUMPY VERSION: {np.__version__}\");\nimport scipy; print(f\"\\t\\t– SCIPY VERSION: {scipy.__version__}\");\n\n# Built In Imports\nfrom collections import Counter\nfrom datetime import datetime\nimport multiprocessing\nfrom glob import glob\nimport warnings\nimport requests\nimport imageio\nimport IPython\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport math\nimport tqdm\nimport time\nimport gzip\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nimport plotly\nimport PIL\nimport cv2\nimport ast\n\n# PRESETS\nLBL_NAMES = [\"Nucleoplasm\", \"Nuclear Membrane\", \"Nucleoli\", \"Nucleoli Fibrillar Center\", \"Nuclear Speckles\", \"Nuclear Bodies\", \"Endoplasmic Reticulum\", \"Golgi Apparatus\", \"Intermediate Filaments\", \"Actin Filaments\", \"Microtubules\", \"Mitotic Spindle\", \"Centrosome\", \"Plasma Membrane\", \"Mitochondria\", \"Aggresome\", \"Cytosol\", \"Vesicles\", \"Negative\"]\nINT_2_STR = {x:LBL_NAMES[x] for x in np.arange(19)}\nINT_2_STR_LOWER = {k:v.lower().replace(\" \", \"_\") for k,v in INT_2_STR.items()}\nSTR_2_INT_LOWER = {v:k for k,v in INT_2_STR_LOWER.items()}\nSTR_2_INT = {v:k for k,v in INT_2_STR.items()}\nFIG_FONT = dict(family=\"Helvetica, Arial\", size=14, color=\"#7f7f7f\")\nLABEL_COLORS = [px.colors.label_rgb(px.colors.convert_to_RGB_255(x)) for x in sns.color_palette(\"Spectral\", len(LBL_NAMES))]\nLABEL_COL_MAP = {str(i):x for i,x in enumerate(LABEL_COLORS)}\n\nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":77.315718,"end_time":"2021-01-31T11:45:56.057286","exception":false,"start_time":"2021-01-31T11:44:38.741568","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

1  BACKGROUND INFORMATION     ⤒

","metadata":{"papermill":{"duration":0.019977,"end_time":"2021-01-31T11:45:56.098171","exception":false,"start_time":"2021-01-31T11:45:56.078194","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

1.1 APPROACH OVERVIEW

\n\n---\n\n**TRAINING**\n\n1. Identify slide-level images containing only one label\n2. Segment slide-level images (get RLEs for all cells in all applicable slide-level images)\n3. Crop RGBY image around each cell\n4. Pad each RGBY tile to be square\n5. Resize each RGBY tile to be (256px by 256px)\n6. **Filter the images based on certain additional factors to obtain a better training dataset -- TBD**\n7. Seperate the channels and store as seperate datasets\n8. Update the dataset (greatly increase the number of negative class examples)\n9. Augment the dataset (rotation, flipping (horizontal and vertical), minor-skew)\n10. Train a model to classify these tile-level images accurately","metadata":{}},{"cell_type":"markdown","source":"

1.2 VISUAL HELPER

\n\n---","metadata":{}},{"cell_type":"markdown","source":"\n\n![basic_idea_graph](https://i.ibb.co/y6YfBzN/basic-idea.png)","metadata":{}},{"cell_type":"markdown","source":"

2  NOTEBOOK SETUP    ⤒

","metadata":{"papermill":{"duration":0.020391,"end_time":"2021-01-31T11:45:56.263882","exception":false,"start_time":"2021-01-31T11:45:56.243491","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Define the path to the root data directory\nROOT_DIR = \"/kaggle/input\"\n\n# Define the path to the competition data directory\nCOMP_DIR = os.path.join(ROOT_DIR, \"hpa-single-cell-image-classification\")\n\n# Define path to the filtered TP IDs for each class\nPKL_DIR = os.path.join(ROOT_DIR, \"hpa-rule-based-single-cell-filtering\")\n\n# Define the paths to the training tiles for the cell-wise classification dataset\nRED_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-red-cell-tile-dataset\")\nGREEN_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-green-cell-tile-dataset\")\nBLUE_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-blue-cell-tile-dataset\")\nYELLOW_TILE_DIR = os.path.join(ROOT_DIR, \"human-protein-atlas-yellow-cell-tile-dataset\")\n\n# Define the paths to the training and testing tfrecord and \n# image folders respectively for the competition data\nTRAIN_IMG_DIR = os.path.join(COMP_DIR, \"train\")\nTRAIN_TFREC_DIR = os.path.join(COMP_DIR, \"train_tfrecords\")\nTEST_IMG_DIR = os.path.join(COMP_DIR, \"test\")\nTEST_TFREC_DIR = os.path.join(COMP_DIR, \"test_tfrecords\")\n\n# Capture all the relevant full image paths for the competition dataset\nTRAIN_IMG_PATHS = sorted([os.path.join(TRAIN_IMG_DIR, f_name) for f_name in os.listdir(TRAIN_IMG_DIR)])\nTEST_IMG_PATHS = sorted([os.path.join(TEST_IMG_DIR, f_name) for f_name in os.listdir(TEST_IMG_DIR)])\nprint(f\"\\n... Recall that 4 training images compose one example (R,G,B,Y) ...\")\nprint(f\"... \\t– i.e. The first 4 training files are:\")\nfor path in [x.rsplit('/',1)[1] for x in TRAIN_IMG_PATHS[:4]]: print(f\"... \\t\\t– {path}\")\nprint(f\"\\n... The number of training images is {len(TRAIN_IMG_PATHS)} i.e. {len(TRAIN_IMG_PATHS)//4} 4-channel images ...\")\nprint(f\"... The number of testing images is {len(TEST_IMG_PATHS)} i.e. {len(TEST_IMG_PATHS)//4} 4-channel images ...\")\n\n# Capture all the relevant full tfrec paths\nTRAIN_TFREC_PATHS = sorted([os.path.join(TRAIN_TFREC_DIR, f_name) for f_name in os.listdir(TRAIN_TFREC_DIR)])\nTEST_TFREC_PATHS = sorted([os.path.join(TEST_TFREC_DIR, f_name) for f_name in os.listdir(TEST_TFREC_DIR)])\nprint(f\"\\n... The number of training tfrecord files is {len(TRAIN_TFREC_PATHS)} ...\")\nprint(f\"... The number of testing tfrecord files is {len(TEST_TFREC_PATHS)} ...\\n\")\n\n# Random Useful Info\nORIGINAL_DIST_MAP = {0: 37472, 1: 4845, 2: 12672, 3: 12882, 4: 17527, 5: 15337, 6: 10198, 7: 18825, 8: 11194, 9: 5322, 10: 7789, 11: 10, 12: 13952, 13: 21168, 14: 27494, 15: 2275, 16: 22738, 17: 5619, 18: 952}\n\n# Define paths to the relevant csv files\nTRAIN_CSV = os.path.join(ROOT_DIR, \"hpa-train-data-with-additional-metadata/updated_train.csv\")\n\nprint(\"\\n... Loading massive train dataframe ...\\n\")\n# Create the relevant dataframe objects\ntrain_df = pd.read_csv(TRAIN_CSV)\n# train_df.mask_rles = train_df.mask_rles.apply(lambda x: ast.literal_eval(x))\n# train_df.mask_bboxes = train_df.mask_bboxes.apply(lambda x: ast.literal_eval(x))\n \nprint(\"\\n\\nTRAIN DATAFRAME\\n\\n\")\ndisplay(train_df.head(3))","metadata":{"papermill":{"duration":1.893314,"end_time":"2021-01-31T11:45:58.178032","exception":false,"start_time":"2021-01-31T11:45:56.284718","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

3  HELPER FUNCTIONS    ⤒

","metadata":{"papermill":{"duration":0.022604,"end_time":"2021-01-31T11:45:58.224086","exception":false,"start_time":"2021-01-31T11:45:58.201482","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def load_image_scaled(img_id, img_dir, img_size=512, load_style=\"tf\"):\n \"\"\" Load An Image Using ID and Directory Path - Composes 4 Individual Images \"\"\"\n def __load_with_tf(path, img_size=512):\n img = tf.io.read_file(path)\n img = tf.image.decode_png(img, channels=1)\n return tf.image.resize(img, (img_size, img_size))[..., 0]\n \n def __load_with_pil(path, img_size=512):\n img = Image.open(path)\n img = img.resize((img_size, img_size))\n return np.asarray(img)\n \n def __load_with_cv2(path, img_size=512):\n img = cv2.imread(path, 0)\n img = cv2.resize(img, (img_size, img_size))\n return img\n \n if load_style is \"tf\":\n load_fn = __load_with_tf\n elif load_style is \"PIL\":\n load_fn = __load_with_pil\n else:\n load_fn = __load_with_cv2\n \n return np.stack(\n [np.asarray(load_fn(os.path.join(img_dir, img_id+f\"_{c}.png\"), img_size)/255.) for c in [\"red\", \"yellow\", \"blue\"]], axis=2\n )\n\n\ndef decode_img(img, img_size=(224,224)):\n \"\"\"TBD\"\"\"\n \n # convert the compressed string to a 3D uint8 tensor\n img = tf.image.decode_png(img, channels=1)\n\n # resize the image to the desired size\n return tf.cast(tf.image.resize(img, img_size), tf.uint8)\n\n\ndef get_color_path_maps(color_dirs, tp_id_map):\n c_p_maps = [{k:[] for k in INT_2_STR.keys()} for _ in range(len(color_dirs))]\n color_d_paths = [\n [d_path for d_path in os.listdir(color_dir) if d_path.endswith(\"_256\")] \\\n for color_dir in color_dirs\n ]\n for c in tqdm(color_d_paths[0], total=len(color_d_paths[0])):\n \n # Get class stuff\n cls = c.split(\"_\", 1)[1].rsplit(\"_\",1)[0]\n cls_idx = STR_2_INT_LOWER[cls]\n \n # Get the relevant color directories\n c_dirs = [\n os.path.join(color_dir, c.replace(\"red\", clr), \"data\", \"train_tiles\", cls) \\\n for clr, color_dir in zip([\"red\", \"green\", \"blue\", \"yellow\"], color_dirs)\n ]\n\n # Update map\n for f_name in tqdm(os.listdir(c_dirs[0]), total=len(os.listdir(c_dirs[0]))):\n # get the relevant full paths\n full_paths = [os.path.join(c_dir, f_name.replace(\"red\", clr)) for clr, c_dir in zip([\"red\", \"green\", \"blue\", \"yellow\"], c_dirs)]\n if tp_id_map==None:\n for c_p_map, full_path in zip(c_p_maps, full_paths):\n c_p_map[cls_idx].append(full_path)\n elif (f_name.endswith(\".png\") and (\"negative\" in full_paths[0] or f_name.rsplit(\"_\", 1)[0] in tp_id_map[cls_idx])):\n for c_p_map, full_path in zip(c_p_maps, full_paths):\n c_p_map[cls_idx].append(full_path)\n else:\n for c_p_map, full_path in zip(c_p_maps, full_paths):\n c_p_map[STR_2_INT[\"Negative\"]].append(full_path)\n return [{k:sorted(v) for k,v in c_p_map.items()} for c_p_map in c_p_maps]\n\n\ndef get_tp_id_map(pkl_dir):\n \"\"\" TBD \"\"\"\n # Capture all relevant paths\n pkl_paths = [\n os.path.join(pkl_dir, f_name) \\\n for f_name in os.listdir(pkl_dir) \\\n if f_name.endswith(\".pkl\")\n ]\n \n # REMOVE AFTER UPDATING CLASSBASED NOTEBOOK\n pkl_paths.append(\"/kaggle/input/tmp-intermediate-filaments-pkl-file/intermediate_filaments_tp_list.pkl\")\n \n # Initialize\n tp_id_map = {}\n for path in pkl_paths:\n class_id = STR_2_INT_LOWER[path.rsplit(\"/\", 1)[1].replace(\"_tp_list.pkl\", \"\")]\n with open(path, \"rb\") as f:\n tp_id_map[class_id] = pickle.load(f)\n return tp_id_map\n\n \ndef plot_rgb(arr, figsize=(12,12)):\n \"\"\" Plot 3 Channel Microscopy Image \"\"\"\n plt.figure(figsize=figsize)\n plt.title(f\"RGB Composite Image\", fontweight=\"bold\")\n plt.imshow(arr)\n plt.axis(False)\n plt.show() \n\n \ndef convert_rgby_to_rgb(arr):\n \"\"\" Convert a 4 channel (RGBY) image to a 3 channel RGB image.\n \n Advice From Competition Host/User: lnhtrang\n\n For annotation (by experts) and for the model, I guess we agree that individual \n channels with full range px values are better. \n In annotation, we toggled the channels. \n For visualization purpose only, you can try blending the channels. \n For example, \n - red = red + yellow\n - green = green + yellow/2\n - blue=blue.\n \n Args:\n arr (numpy array): The RGBY, 4 channel numpy array for a given image\n \n Returns:\n RGB Image\n \"\"\"\n \n rgb_arr = np.zeros_like(arr[..., :-1])\n rgb_arr[..., 0] = arr[..., 0]\n rgb_arr[..., 1] = arr[..., 1]+arr[..., 3]/2\n rgb_arr[..., 2] = arr[..., 2]\n \n return rgb_arr\n \n \ndef plot_ex(arr, figsize=(20,6), title=None, plot_merged=True, rgb_only=False):\n \"\"\" Plot 4 Channels Side by Side \"\"\"\n if plot_merged and not rgb_only:\n n_images=5 \n elif plot_merged and rgb_only:\n n_images=4\n elif not plot_merged and rgb_only:\n n_images=4\n else:\n n_images=3\n plt.figure(figsize=figsize)\n if type(title) == str:\n plt.suptitle(title, fontsize=20, fontweight=\"bold\")\n\n for i, c in enumerate([\"Red Channel – Microtubles\", \"Green Channel – Protein of Interest\", \"Blue - Nucleus\", \"Yellow – Endoplasmic Reticulum\"]):\n if not rgb_only:\n ch_arr = np.zeros_like(arr[..., :-1]) \n else:\n ch_arr = np.zeros_like(arr)\n if c in [\"Red Channel – Microtubles\", \"Green Channel – Protein of Interest\", \"Blue - Nucleus\"]:\n ch_arr[..., i] = arr[..., i]\n else:\n if rgb_only:\n continue\n ch_arr[..., 0] = arr[..., i]\n ch_arr[..., 1] = arr[..., i]\n plt.subplot(1,n_images,i+1)\n plt.title(f\"{c.title()}\", fontweight=\"bold\")\n plt.imshow(ch_arr)\n plt.axis(False)\n \n if plot_merged:\n plt.subplot(1,n_images,n_images)\n \n if rgb_only:\n plt.title(f\"Merged RGB\", fontweight=\"bold\")\n plt.imshow(arr)\n else:\n plt.title(f\"Merged RGBY into RGB\", fontweight=\"bold\")\n plt.imshow(convert_rgby_to_rgb(arr))\n plt.axis(False)\n \n plt.tight_layout(rect=[0, 0.2, 1, 0.97])\n plt.show()\n \n \ndef flatten_list_of_lists(l_o_l):\n return [item for sublist in l_o_l for item in sublist]\n\n\ndef create_input_list(crp, cgp, cbp, cyp, shuffle=True, val_split=0.025):\n lbl_arr = flatten_list_of_lists([[k,]*len(v) for k, v in sorted(crp.items())])\n cr_arr = flatten_list_of_lists([v for k,v in sorted(crp.items())])\n cg_arr = flatten_list_of_lists([v for k,v in sorted(cgp.items())])\n cb_arr = flatten_list_of_lists([v for k,v in sorted(cbp.items())])\n cy_arr = flatten_list_of_lists([v for k,v in sorted(cyp.items())])\n \n if val_split is not None:\n val_lbl_arr = lbl_arr[:int(len(lbl_arr)*val_split)]\n lbl_arr = lbl_arr[int(len(lbl_arr)*val_split):]\n \n val_cr_arr = cr_arr[:int(len(cr_arr)*val_split)]\n cr_arr = cr_arr[int(len(cr_arr)*val_split):]\n \n val_cg_arr = cg_arr[:int(len(cg_arr)*val_split)]\n cg_arr = cg_arr[int(len(cg_arr)*val_split):]\n \n val_cb_arr = cb_arr[:int(len(cb_arr)*val_split)]\n cb_arr = cb_arr[int(len(cb_arr)*val_split):]\n\n val_cy_arr = cy_arr[:int(len(cy_arr)*val_split)]\n cy_arr = cy_arr[int(len(cy_arr)*val_split):]\n \n if shuffle:\n to_shuffle = list(zip(cr_arr, cg_arr, cb_arr, cy_arr, lbl_arr))\n random.shuffle(to_shuffle)\n cr_arr, cg_arr, cb_arr, cy_arr, lbl_arr = zip(*to_shuffle)\n \n if val_split is not None:\n val_to_shuffle = list(zip(val_cr_arr, val_cg_arr, val_cb_arr, val_cy_arr, val_lbl_arr))\n random.shuffle(val_to_shuffle)\n val_cr_arr, val_cg_arr, val_cb_arr, val_cy_arr, val_lbl_arr = zip(*val_to_shuffle)\n \n if val_split is None:\n return list(cr_arr), list(cg_arr), list(cb_arr), list(cy_arr), list(lbl_arr)\n else:\n return (list(cr_arr), list(cg_arr), list(cb_arr), list(cy_arr), list(lbl_arr)), \\\n (list(val_cr_arr), list(val_cg_arr), list(val_cb_arr), list(val_cy_arr), list(val_lbl_arr))\n\n\ndef get_class_wts(single_ch_paths, n_classes=19, exclude_mitotic=True, multiplier=10, return_counts=False):\n \"\"\" TBD \"\"\"\n # Get class counts\n class_counts = {c_idx:len(single_ch_paths[c_idx]) for c_idx in range(n_classes)}\n\n # Exclude mitotic spindle\n if exclude_mitotic:\n real_min_count = list(sorted(class_counts.values(), reverse=True))[-2]\n else:\n real_min_count = list(sorted(class_counts.values(), reverse=True))[-1]\n\n # Calculate weights\n class_wts = {k:min(1, multiplier*(real_min_count/v)) for k,v in class_counts.items()}\n\n if exclude_mitotic:\n # Manually adjust mitotic spindle to a more appropriate value\n class_wts[min(class_counts, key=class_counts.get)] = 1.0\n\n if return_counts:\n return class_wts, class_counts\n else:\n return class_wts","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","papermill":{"duration":3.049332,"end_time":"2021-01-31T11:46:01.296459","exception":false,"start_time":"2021-01-31T11:45:58.247127","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4  PREPARING THE DATASET - TF.DATA    ⤒

\n\nThis section will explore how to use tf.data to setup the input pipeline","metadata":{"papermill":{"duration":0.026161,"end_time":"2021-01-31T11:46:01.349387","exception":false,"start_time":"2021-01-31T11:46:01.323226","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

4.0 INITIALIZE DICTIONARIES MAPPING PATHS TO RELEVANT FILES

\n\n---","metadata":{}},{"cell_type":"code","source":"TILE_DIRS = [RED_TILE_DIR, GREEN_TILE_DIR, BLUE_TILE_DIR, YELLOW_TILE_DIR]\n# TP_ID_MAP = get_tp_id_map(PKL_DIR)\n\n# Define the paths to the training files for the tile dataset as a map from class index to list of paths\nRED_FILE_MAP, GREEN_FILE_MAP, BLUE_FILE_MAP, YELLOW_FILE_MAP = \\\n get_color_path_maps(TILE_DIRS, None)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"\\n... Comparison Before and After Manual Heuristic Filtering Function ...\\n\")\n# for i in range(19): print(f\"CLS {i:>2}\\n --> ORIGINAL DISTRIBUTION - {ORIGINAL_DIST_MAP[i]}\\n --> NEW DISTRIBUTION - {len(RED_FILE_MAP[i])}\\n\") ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.1 TRANSFORM THE PATH MAP INTO LISTS FOR INPUT INTO TF.DATA

\n\n---\n\nUsing the command **`tf.data.Dataset.list_files`** unfortunately either shuffles each list individually or forces the files into a sorted order. We want the files shuffled, but we want the shuffling to be identical across the various colour channels.\n\nWe also take this opportunity to identify the labels for each image and create an array as such.","metadata":{"papermill":{"duration":0.02683,"end_time":"2021-01-31T11:46:01.403131","exception":false,"start_time":"2021-01-31T11:46:01.376301","status":"completed"},"tags":[]}},{"cell_type":"code","source":"VAL_FRAC = 0.075\n\n# red_inputs, green_inputs, blue_inputs, yellow_inputs, labels\ntrain_inputs, val_inputs = create_input_list(\n RED_FILE_MAP, \n GREEN_FILE_MAP, \n BLUE_FILE_MAP, \n YELLOW_FILE_MAP, \n shuffle=True,\n val_split=VAL_FRAC,\n)","metadata":{"papermill":{"duration":0.153587,"end_time":"2021-01-31T11:46:01.584256","exception":false,"start_time":"2021-01-31T11:46:01.430669","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.2 DETERMINE AN APPROPRIATE CLASS WEIGHTING

\n\n---\n\nI was using this previously. I will still calculate it here, but I won't be adding it to the fit function.","metadata":{"papermill":{"duration":0.028363,"end_time":"2021-01-31T11:46:01.641042","exception":false,"start_time":"2021-01-31T11:46:01.612679","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# class_wts, class_cnts = get_class_wts(RED_FILE_MAP, return_counts=True, multiplier=23.203)\nclass_wts, class_cnts = get_class_wts(RED_FILE_MAP, return_counts=True, multiplier=50)\nprint(\"\\n ... CLASSWISE COUNTS ... \\n\")\ndisplay(class_cnts)\n\nprint(\"\\n ... CLASS WEIGHTING ... \\n\")\ndisplay(class_wts)","metadata":{"papermill":{"duration":1.172573,"end_time":"2021-01-31T11:46:02.842773","exception":false,"start_time":"2021-01-31T11:46:01.6702","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.3 Define Model/Dataset Parameters

\n\n---","metadata":{"papermill":{"duration":0.03055,"end_time":"2021-01-31T11:46:02.903692","exception":false,"start_time":"2021-01-31T11:46:02.873142","status":"completed"},"tags":[]}},{"cell_type":"code","source":"### POTENTIAL LOSS FN ###\n# def macro_double_soft_f1(y, y_hat):\n# \"\"\"Compute the macro soft F1-score as a cost (average 1 - soft-F1 across all labels).\n# Use probability values instead of binary predictions.\n# This version uses the computation of soft-F1 for both positive and negative class for each label.\n \n# Args:\n# y (int32 Tensor): targets array of shape (BATCH_SIZE, N_LABELS)\n# y_hat (float32 Tensor): probability matrix from forward propagation of shape (BATCH_SIZE, N_LABELS)\n \n# Returns:\n# cost (scalar Tensor): value of the cost function for the batch\n# \"\"\"\n# y = tf.cast(y, tf.float32)\n# y_hat = tf.cast(y_hat, tf.float32)\n# tp = tf.reduce_sum(y_hat * y, axis=0)\n# fp = tf.reduce_sum(y_hat * (1 - y), axis=0)\n# fn = tf.reduce_sum((1 - y_hat) * y, axis=0)\n# tn = tf.reduce_sum((1 - y_hat) * (1 - y), axis=0)\n# soft_f1_class1 = 2*tp / (2*tp + fn + fp + 1e-16)\n# soft_f1_class0 = 2*tn / (2*tn + fn + fp + 1e-16)\n# cost_class1 = 1 - soft_f1_class1 # reduce 1 - soft-f1_class1 in order to increase soft-f1 on class 1\n# cost_class0 = 1 - soft_f1_class0 # reduce 1 - soft-f1_class0 in order to increase soft-f1 on class 0\n# cost = 0.5 * (cost_class1 + cost_class0) # take into account both class 1 and class 0\n# macro_cost = tf.reduce_mean(cost) # average on all labels\n# return macro_cost\n\n### POTENTIAL LOSS FN ###\n# def macro_soft_f1(y, y_hat):\n# \"\"\"Compute the macro soft F1-score as a cost.\n# Average (1 - soft-F1) across all labels.\n# Use probability values instead of binary predictions.\n \n# Args:\n# y (int32 Tensor): targets array of shape (BATCH_SIZE, N_LABELS)\n# y_hat (float32 Tensor): probability matrix of shape (BATCH_SIZE, N_LABELS)\n \n# Returns:\n# cost (scalar Tensor): value of the cost function for the batch\n# \"\"\"\n \n# y = tf.cast(y, tf.float32)\n# y_hat = tf.cast(y_hat, tf.float32)\n# tp = tf.reduce_sum(y_hat * y, axis=0)\n# fp = tf.reduce_sum(y_hat * (1 - y), axis=0)\n# fn = tf.reduce_sum((1 - y_hat) * y, axis=0)\n# soft_f1 = 2*tp / (2*tp + fn + fp + 1e-16)\n# cost = 1 - soft_f1 # reduce 1 - soft-f1 in order to increase soft-f1\n# macro_cost = tf.reduce_mean(cost) # average on all labels\n \n# return macro_cost\n\n### POTENTIAL METRIC ###\n# def macro_f1(y, y_hat, thresh=0.5):\n# \"\"\"Compute the macro F1-score on a batch of observations (average F1 across labels)\n \n# Args:\n# y (int32 Tensor): labels array of shape (BATCH_SIZE, N_LABELS)\n# y_hat (float32 Tensor): probability matrix from forward propagation of shape (BATCH_SIZE, N_LABELS)\n# thresh: probability value above which we predict positive\n \n# Returns:\n# macro_f1 (scalar Tensor): value of macro F1 for the batch\n# \"\"\"\n# y_pred = tf.cast(tf.greater(y_hat, thresh), tf.float32)\n# tp = tf.cast(tf.math.count_nonzero(y_pred * y, axis=0), tf.float32)\n# fp = tf.cast(tf.math.count_nonzero(y_pred * (1 - y), axis=0), tf.float32)\n# fn = tf.cast(tf.math.count_nonzero((1 - y_pred) * y, axis=0), tf.float32)\n# f1 = 2*tp / (2*tp + fn + fp + 1e-16)\n# macro_f1 = tf.reduce_mean(f1)\n# return macro_f1\n\n\n# Using an LR ramp up because fine-tuning a pre-trained model.\n# Starting with a high LR would break the pre-trained weights.\nN_EPOCHS=10\nLR_START = 0.0005\nLR_MAX = 0.0011\nLR_MIN = 0.0005\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 2\nLR_EXP_DECAY = 0.75\n\ndef lrfn(epoch):\n if epoch < LR_RAMPUP_EPOCHS:\n lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n lr = LR_MAX\n else:\n lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n return lr\n\n# VIEW SCHEDULE\nrng = [i for i in range(N_EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nplt.figure(figsize=(10,4))\nplt.plot(rng, y)\nplt.title(\"CUSTOM LR SCHEDULE\", fontweight=\"bold\")\nplt.show()\n\nprint(f\"Learning rate schedule: {y[0]:.3g} to {max(y):.3g} to {y[-1]:.3g}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#PARAMS\nMODEL_CKPT_DIR = \"/kaggle/working/ebnet_b2_wdensehead\"\nDROP_YELLOW = True\nNO_NEG_CLASS = False\n\nif NO_NEG_CLASS:\n class_wts = {k:v for k,v in class_wts.items() if k!=18}\n class_cnts = {k:v for k,v in class_cnts.items() if k!=18}\n n_classes = 18\nelse:\n n_classes=19\n \nBATCH_SIZE=32\nOPTIMIZER = tf.keras.optimizers.Adam(lr=LR_START)\nLOSS_FN = \"binary_crossentropy\"\nSHUFF_BUFF = 500\n\n\n# AUTO-CALCULATED\nN_EX = len(RED_FILE_MAP[0])\nN_VAL = int(VAL_FRAC*N_EX)\nN_TRAIN = N_EX-N_VAL\n\nif not os.path.isdir(MODEL_CKPT_DIR):\n os.makedirs(MODEL_CKPT_DIR, exist_ok=True)\n \nprint(f\"{N_TRAIN:<7} TRAINING EXAMPLES\")\nprint(f\"{N_VAL:<7} VALIDATION EXAMPLES\")","metadata":{"papermill":{"duration":0.148614,"end_time":"2021-01-31T11:46:03.082836","exception":false,"start_time":"2021-01-31T11:46:02.934222","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.4 CREATE THE TRAINING AND VALIDATION DATASETS

\n\n---\n\nWIP","metadata":{"papermill":{"duration":0.030494,"end_time":"2021-01-31T11:46:03.144496","exception":false,"start_time":"2021-01-31T11:46:03.114002","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# TRAIN DATASET\ntrain_path_ds = tf.data.Dataset.zip(\n tuple([tf.data.Dataset.from_tensor_slices(input_ds) for input_ds in train_inputs])\n)\n\n# VALIDATION DATASET\nval_path_ds = tf.data.Dataset.zip(\n tuple([tf.data.Dataset.from_tensor_slices(input_ds) for input_ds in val_inputs])\n)\n\nprint(f\"\\n ... THERE ARE {N_EX} CELL TILES IN OUR FULL DATASET ... \")\nprint(f\" ... THERE ARE {N_TRAIN} CELL TILES IN OUR TRAIN DATASET ... \")\nprint(f\" ... THERE ARE {N_VAL} CELL TILES IN OUR VALIDATION DATASET ... \\n\")\n\nprint(train_path_ds)\n\nfor a,b,c,d,e in train_path_ds.take(1): \n print(f\"\\tRed Path --> {a}\\n\\t\" \\\n f\"Green Path --> {b}\\n\\t\" \\\n f\"Blue Path --> {c}\\n\\t\" \\\n f\"Yellow Path --> {d}\\n\\t\" \\\n f\"Example Label --> {e} ({INT_2_STR[e.numpy()]})\\n\")","metadata":{"papermill":{"duration":0.307114,"end_time":"2021-01-31T11:46:03.482374","exception":false,"start_time":"2021-01-31T11:46:03.17526","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

4.5 PREPROCESS THE DATASETS (COMBINE RGBY) AND ONE-HOT-ENCODE LABELS

\n\n---\n\nWe also augment, visualize, shuffle, batch, and prefetch.\n\n\n","metadata":{"papermill":{"duration":0.04616,"end_time":"2021-01-31T11:46:03.575519","exception":false,"start_time":"2021-01-31T11:46:03.529359","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def preprocess_path_ds(rp, gp, bp, yp, lbl, img_size=(224,224), combine=True, drop_yellow=True, no_neg=True):\n \"\"\" TBD \"\"\"\n \n # Adjust class output expectation\n if no_neg:\n if lbl==18:\n lbl_arr = tf.zeros((18,), dtype=tf.uint8)\n else:\n lbl_arr = tf.one_hot(lbl, 18, dtype=tf.uint8)\n else:\n lbl_arr = tf.one_hot(lbl, 19, dtype=tf.uint8)\n \n ri = decode_img(tf.io.read_file(rp), img_size)\n gi = decode_img(tf.io.read_file(gp), img_size)\n bi = decode_img(tf.io.read_file(bp), img_size)\n\n if combine and drop_yellow:\n return tf.stack([ri[..., 0], gi[..., 0], bi[..., 0]], axis=-1), lbl_arr\n elif combine:\n yi = decode_img(tf.io.read_file(yp), img_size)\n return tf.stack([ri[..., 0], gi[..., 0], bi[..., 0], yi[..., 0]], axis=-1), lbl_arr\n elif drop_yellow:\n return ri, gi, bi, lbl_arr\n else:\n yi = decode_img(tf.io.read_file(yp), img_size)\n return ri, gi, bi, yi, lbl_arr\n \n\ndef augment(img_batch, lbl_batch):\n # SEEDING & KERNEL INIT\n K = tf.random.uniform((1,), minval=0, maxval=4, dtype=tf.dtypes.int32)[0]\n \n img_batch = tf.image.random_flip_left_right(img_batch)\n img_batch = tf.image.random_flip_up_down(img_batch)\n img_batch = tf.image.rot90(img_batch, K)\n \n img_batch = tf.image.random_saturation(img_batch, 0.85, 1.15)\n img_batch = tf.image.random_brightness(img_batch, 0.1)\n img_batch = tf.image.random_contrast(img_batch, 0.85, 1.15)\n \n # Can't figure this out right now\n # # Apply a random crop\n # img_batch = tf.where(K==0, tf.map_fn(\n # fn=lambda img: tf.image.resize(tf.image.random_crop(tf.cast(img, tf.float32), (192,192,3)), (224,224)),\n # elems=img_batch,), img_batch)\n\n return img_batch, lbl_batch","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_CACHE_DIR = \"/kaggle/train_cache\"\nVAL_CACHE_DIR = \"/kaggle/val_cache\"\n\nif not os.path.isdir(TRAIN_CACHE_DIR):\n os.makedirs(TRAIN_CACHE_DIR, exist_ok=True)\nif not os.path.isdir(VAL_CACHE_DIR):\n os.makedirs(VAL_CACHE_DIR, exist_ok=True)\n\ntrain_ds = train_path_ds.map(\n lambda r,g,b,y,l: preprocess_path_ds(r,g,b,y,l, drop_yellow=DROP_YELLOW, no_neg=NO_NEG_CLASS), \n num_parallel_calls=tf.data.AUTOTUNE\n)\nval_ds = val_path_ds.map(\n lambda r,g,b,y,l: preprocess_path_ds(r,g,b,y,l, drop_yellow=DROP_YELLOW, no_neg=NO_NEG_CLASS), \n num_parallel_calls=tf.data.AUTOTUNE\n)\n\n# VISUALIZE EXAMPLES\nprint(\"\\n\\t\\t... TRAIN EXAMPLES ...\\n\")\nfor x,y in train_ds.take(3):\n if y.numpy().sum()==0:\n title_str = INT_2_STR[18]\n else:\n title_str = INT_2_STR[np.argmax(y.numpy())]\n plot_ex(x.numpy(), title=f\"{title_str}\", rgb_only=DROP_YELLOW)\n\nprint(\"\\n\\t\\t... VAL EXAMPLES ...\\n\")\nfor x,y in val_ds.take(3):\n if y.numpy().sum()==0:\n title_str = INT_2_STR[18]\n else:\n title_str = INT_2_STR[np.argmax(y.numpy())]\n plot_ex(x.numpy(), title=f\"{title_str}\", rgb_only=DROP_YELLOW)\n \ntrain_ds = train_ds.cache(TRAIN_CACHE_DIR) \\\n .shuffle(SHUFF_BUFF) \\\n .batch(BATCH_SIZE) \\\n .map(augment, num_parallel_calls=tf.data.AUTOTUNE) \\\n .prefetch(tf.data.AUTOTUNE)\n\nval_ds = val_ds.cache(VAL_CACHE_DIR) \\\n .batch(BATCH_SIZE) \\\n .prefetch(tf.data.AUTOTUNE)\n\n# VISUALIZE EXAMPLES\nprint(\"\\n\\t\\t... TRAIN EXAMPLES POST AUGMENTATION ...\\n\")\nfor xs,ys in train_ds.take(1):\n for i, (x,y) in enumerate(zip(xs, ys)):\n if i==4:\n break\n if y.numpy().sum()==0:\n title_str = INT_2_STR[18]\n else:\n title_str = INT_2_STR[np.argmax(y.numpy())]\n plot_ex(x.numpy().astype(np.uint8), title=f\"{title_str}\", rgb_only=DROP_YELLOW)","metadata":{"papermill":{"duration":4.014693,"end_time":"2021-01-31T11:46:07.636753","exception":false,"start_time":"2021-01-31T11:46:03.62206","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5  MODELLING    ⤒

\n\nThis section will explore how to acquire, augment and train the model","metadata":{"papermill":{"duration":0.026161,"end_time":"2021-01-31T11:46:01.349387","exception":false,"start_time":"2021-01-31T11:46:01.323226","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"

5.1 LOAD THE MODEL BACKBONE FOR TRAINING

\n\n---\n\nWIP","metadata":{"papermill":{"duration":0.02683,"end_time":"2021-01-31T11:46:01.403131","exception":false,"start_time":"2021-01-31T11:46:01.376301","status":"completed"},"tags":[]}},{"cell_type":"code","source":" def get_backbone(efficientnet_name=\"efficientnet_b0\", input_shape=(224,224,3), include_top=False, weights=\"imagenet\", pooling=\"avg\"):\n if \"b0\" in efficientnet_name:\n eb = tf.keras.applications.EfficientNetB0(\n include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n )\n elif \"b1\" in efficientnet_name:\n eb = tf.keras.applications.EfficientNetB1(\n include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n )\n elif \"b2\" in efficientnet_name:\n eb = tf.keras.applications.EfficientNetB2(\n include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n )\n elif \"b3\" in efficientnet_name:\n eb = tf.keras.applications.EfficientNetB3(\n include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n )\n elif \"b4\" in efficientnet_name:\n eb = tf.keras.applications.EfficientNetB4(\n include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n )\n elif \"b5\" in efficientnet_name:\n eb = tf.keras.applications.EfficientNetB5(\n include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n )\n elif \"b6\" in efficientnet_name:\n eb = tf.keras.applications.EfficientNetB6(\n include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n )\n elif \"b7\" in efficientnet_name:\n eb = tf.keras.applications.EfficientNetB7(\n include_top=include_top, weights=weights, pooling=pooling, input_shape=input_shape\n )\n else:\n raise ValueError(\"Invalid EfficientNet Name!!!\")\n return eb\n\n\n def add_head_to_bb(bb, n_classes=19, dropout=0.05, head_layer_nodes=(512,)):\n x = tf.keras.layers.BatchNormalization()(bb.output)\n x = tf.keras.layers.Dropout(dropout)(x)\n \n for n_nodes in head_layer_nodes:\n x = tf.keras.layers.Dense(n_nodes, activation=\"relu\")(x)\n x = tf.keras.layers.BatchNormalization()(x)\n x = tf.keras.layers.Dropout(dropout/2)(x)\n \n output = tf.keras.layers.Dense(n_classes, activation=\"sigmoid\")(x)\n return tf.keras.Model(inputs=bb.inputs, outputs=output)\n eb = get_backbone(\"b2\")\n eb = add_head_to_bb(eb, n_classes, dropout=0.5)\n eb.compile(optimizer=OPTIMIZER, loss=LOSS_FN, metrics=[\"acc\", tf.keras.metrics.AUC(name=\"auc\", multi_label=True)])\n\n tf.keras.utils.plot_model(eb, show_shapes=True, show_dtype=True, dpi=55)\n\n#eb = tf.keras.models.load_model(\"../input/hpa-cellwise-classification-training/ebnet_b2_wdensehead/ckpt-0009-0.0801.ckpt\")","metadata":{"_kg_hide-output":false,"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#eb = tf.keras.models.load_model(\"../input/hpa-cellwise-classification-training/ebnet_b2_wdensehead/ckpt-0009-0.0801.ckpt\")","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.2 FIT THE MODEL

\n\n---\n\nWIP","metadata":{}},{"cell_type":"code","source":"history = eb.fit(\n train_ds, \n validation_data=val_ds, \n callbacks=[\n tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, verbose=1, restore_best_weights=True),\n tf.keras.callbacks.ModelCheckpoint(filepath=os.path.join(MODEL_CKPT_DIR, \"ckpt-{epoch:04d}-{val_loss:.4f}.ckpt\"), verbose=1),\n tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=1)\n ], \n class_weight=class_wts, \n epochs=N_EPOCHS\n)\neb.save(\"./model_b2_224_w_densehead_classifier\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

5.3 VISUALIZE TRAINING

\n\n---\n\nWIP","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (19, 12))\n\nplt.subplot(3,1,1)\nplt.xlabel(\"Epochs\", fontweight=\"bold\")\nplt.ylabel(\"Loss\", fontweight=\"bold\")\nplt.plot(history.history[\"loss\"], label = \"Training Loss\" , marker='o')\nplt.plot(history.history[\"val_loss\"], label = \"Validation Loss\" , marker='x')\nplt.grid(True)\nplt.legend()\n\nplt.subplot(3,1,2)\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"Accuracy\")\nplt.plot(history.history[\"acc\"], label = \"Training Accuracy\" , marker='o')\nplt.plot(history.history[\"val_acc\"], label = \"Validation Accuracy\" , marker='x')\nplt.grid(True)\nplt.legend()\n\nplt.subplot(3,1,3)\nplt.xlabel(\"Epochs\")\nplt.ylabel(\"AUC\")\nplt.plot(history.history[\"auc\"], label = \"Training AUC\" , marker='o')\nplt.plot(history.history[\"val_auc\"], label = \"Validation AUC\" , marker='x')\nplt.grid(True)\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}