{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Single-cell classification\n\n## Part ONE: Preprocessing  \n1. create segmentation masks for all images  \n2. extract bounding boxes from segmentation masks  \n3. load img_ids, bboxes, class_labels into one dataframe and save to pickle file  \n\n## Part TWO: Visualization  \n1. create plot of label distribution  \n2. create bar plot of img dimensions  \n3. create bar plot of # of imgs with labels per img  \n4. show some channel-combined sample images from dataset  \n5. show some images with bounding boxes\n6. create color histograms?  \n\n## Part THREE: Single-Cell Multi-Label Classification Network  \n### = \"Which cell type is this cropped cell?\"  \nthis is trained with cropped cells and their labels  \n1. load dataframe from pickle file  \n2. shuffle dataframe  \n3. partition dataframe  \n4. split dataframe into train_df and val_df  \n5. set hyperparameters  \n6. set callbacks: early stopping and checkpoint save\n7. set training visualizations/plots from history  \n8. train network  \n\n## Part FOUR: Binary Classification Network  \n### = \"Where are the cells in this image?\"  \nthis is trained with the bounding boxes and the images  \n1. load dataframe from pickle file  \n2. shuffle dataframe  \n3. partition dataframe  \n4. split dataframe into train_df and val_df  \n5. set hyperparameters  \n6. set callbacks: early stopping and checkpoint save\n7. set training visualizations/plots from history  \n8. train network  \n\n## Part FIVE: Combine Networks\n### build a two stage detector and classifier that creates cell bboxes and then classifies those cells","metadata":{}},{"cell_type":"markdown","source":"# Part ONE: Preprocessing","metadata":{}},{"cell_type":"code","source":"%%script echo skipping\n#constants\nimg_folder_path=\"../input/hpa-single-cell-image-classification/train/\"\ncsv_file_path=\"../input/hpa-single-cell-image-classification/train.csv\"\nmask_folder_path=\"./masks\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LABELS= {\n0: \"Nucleoplasm\",\n1: \"Nuclear membrane\",\n2: \"Nucleoli\",\n3: \"Nucleoli fibrillar center\",\n4: \"Nuclear speckles\",\n5: \"Nuclear bodies\",\n6: \"Endoplasmic reticulum\",\n7: \"Golgi apparatus\",\n8: \"Intermediate filaments\",\n9: \"Actin filaments\",\n10: \"Microtubules\",\n11: \"Mitotic spindle\",\n12: \"Centrosome\",\n13: \"Plasma membrane\",\n14: \"Mitochondria\",\n15: \"Aggresome\",\n16: \"Cytosol\",\n17: \"Vesicles and punctate cytosolic patterns\",\n18: \"Negative\"\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#code to load csv file goes here\n\n#imgid_labels_array: array of all imgids and corresponding labels\n#imgid_array: array of all imgids\n#labels_dict: dictionary with imgids and the corresponding labels\nimport pandas as pd\nid_labels_array=pd.read_csv(csv_file_path)\n#this line is not necessary anymore as the tf pipeline separates the values already\nid_labels_array_separated=id_labels_array[\"Label\"].apply(lambda x:list(map(int, x.split(\"|\"))))\nid_array=(id_labels_array[\"ID\"]).tolist()\nlabels_dict=id_labels_array.set_index('ID').T.to_dict('list')\nlabels_dict = {num: labels[0] for num, labels in labels_dict.items()}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#function to combine rgby to rgb\n#source:https://www.kaggle.com/kwentar/visualization-examples-of-each-class-in-rgb#Load-data:\nimport numpy as np\ndef rgby_to_rgb(r,g,b,y):\n    image_width,image_height=r.size\n    rgb_image = np.zeros(shape=(image_height, image_width, 3), dtype=np.float)\n    yellow = np.array(y)\n    # yellow is red + green\n    rgb_image[:, :, 0] += yellow/2   \n    rgb_image[:, :, 1] += yellow/2\n    # loop for R,G and B channels\n    for index, channel in enumerate([r,g,b]):\n        current_image = channel\n        rgb_image[:, :, index] += current_image\n    # Normalize image\n    rgb_image = rgb_image / rgb_image.max() * 255\n    return rgb_image.astype(np.uint8)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#function to get rgb image from only img_id\nfrom PIL import Image\ndef imgid_to_rgb(img_id):\n    r=Image.open(img_folder_path+img_id+\"_red.png\")\n    g=Image.open(img_folder_path+img_id+\"_green.png\")\n    b=Image.open(img_folder_path+img_id+\"_blue.png\")\n    y=Image.open(img_folder_path+img_id+\"_yellow.png\")\n    rgb=rgby_to_rgb(r,g,b,y)\n    return rgb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#function to convert all images to rgb images\n#exceeds 9h kaggle runtime so i ran this offline\n#these files have been saved in the datasets named hpa-composite-images-x-of-20\nfrom PIL import Image\nimport numpy\nfrom tqdm import tqdm\n\nCOMPOSITE_IMG_PATH=\"./composites/\"\nimport os\nif not os.path.exists(COMPOSITE_IMG_PATH):\n    os.makedirs(COMPOSITE_IMG_PATH)\n\nfor img_id in tqdm(id_array[:5]):\n    img_rgb = Image.fromarray(imgid_to_rgb(img_id))\n    img_rgb.save(COMPOSITE_IMG_PATH+img_id+\".png\")\n    \n#convert the id_labels_array to proper file paths of the composite images\nimport numpy as np\nfrom tqdm import tqdm\nimg_paths={}\nid_array_split=np.array_split(id_array, 20) #the dataset was split into 20 parts\nfor i in range(20):\n    n=i+1\n    for img_id in tqdm(id_array_split[i]):\n        img_paths[img_id]=\"../input/hpa-composite-images-\"+str(n)+\"-of-20/\"+img_id+\".png\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n%%capture\n#function for cell segmentation\n!pip install https://github.com/CellProfiling/HPA-Cell-Segmentation/archive/master.zip\nimport hpacellseg.cellsegmentator as cellsegmentator\nfrom hpacellseg.utils import label_cell, label_nuclei\nfrom PIL import Image\nimport numpy as np\nNUC_MODEL = \"../input/hpacellsegmentatormodelweights/dpn_unet_nuclei_v1.pth\"\nCELL_MODEL = \"../input/hpacellsegmentatormodelweights/dpn_unet_cell_3ch_v1.pth\"\nsegmentator = cellsegmentator.CellSegmentator(\n    NUC_MODEL,\n    CELL_MODEL,\n    scale_factor=0.25,\n    device=\"cpu\",\n    padding=False,\n    multi_channel_model=True,\n)\ndef get_mask(img_id):\n    ch_r=Image.open(img_folder_path+img_id+\"_red.png\")\n    ch_y=Image.open(img_folder_path+img_id+\"_yellow.png\")\n    ch_b=Image.open(img_folder_path+img_id+\"_blue.png\")\n    nuc_segmentations = segmentator.pred_nuclei([np.asarray( ch_b )])\n    cell_segmentations = segmentator.pred_cells([\n            [np.asarray( ch_r )],\n            [np.asarray( ch_y )],\n            [np.asarray( ch_b )]\n        ])\n    nuclei_mask, mask = label_cell(nuc_segmentations[0], cell_segmentations[0])\n    mask = np.uint8(mask)\n    return mask","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#https://www.kaggle.com/paulorzp/rle-functions-run-lenght-encode-decode\n#these are only helper functions\ndef mask2rle(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n \ndef rle2mask(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (width,height) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#use these for the masks of the dataset (they have different cell instances in one mask)\ndef multicell_rlencode(mask):\n    flat_mask=np.ravel(mask)\n    cell_ids=set(flat_mask)\n    cell_ids.remove(0)\n    mask_rle=list()\n    for cell_id in cell_ids:\n        mask_rle.append(mask2rle(np.where(cell_id==mask,1,0)))\n    return mask_rle\ndef multicell_rldecode(mask_rle,shape):\n    mask = np.zeros(shape, dtype=np.uint8)\n    for i,layer in enumerate(mask_rle):\n        mask_decoded=rle2mask(layer, shape)\n        tmp=np.where(mask_decoded==1,i+1,0)\n        mask=np.where(tmp == 0, mask, tmp)\n    return mask","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#function for bbox creation\ndef get_bboxes(mask):\n    mask_flattened=np.ravel(mask)\n    cell_ids=set(mask_flattened)\n    cell_ids.remove(0)\n    bboxes=list()\n    for cell_id in cell_ids:\n        a = np.where(mask == cell_id)\n        ymin, ymax, xmin, xmax = np.min(a[0]), np.max(a[0]), np.min(a[1]), np.max(a[1])\n        bboxes.append([ymin,ymax,xmin,xmax])\n    return bboxes","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\nfrom PIL import Image\ndef get_shape(img_id,img_path):\n    im = Image.open(img_path+\"/\"+img_id+\"_green.png\")\n    shape = im.size\n    return shape\ndef load_mask_from_file(img_id,mask_folder_path):\n    mask_rle=np.load(path+\"/mask_rle_\"+img_id+'.npy', allow_pickle=True)\n    shape=get_shape(img_id,\"../input/hpa-single-cell-image-classification/train\")\n    return multicell_rldecode(mask_rle,shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#iterates through image ids in array id_array:\n#    1.creates mask as rle string and saves as numpy array\n#    2.creates bboxes from mask and adds to dictionary(saved as pickle file)\n#I've done this offline as the process takes longer than the max 9h offered by kaggle\n#the results of this are stored in my dataset on kaggle:\n#    bboxes stored in dataset \"hpa-bboxes\"\n#    rle encoded masks not stored online because too big for proper upload\n\n\nfrom tqdm import tqdm\nimport os\nimport pickle\nbboxes_dict={}\nif not os.path.exists(mask_folder_path):\n    os.makedirs(mask_folder_path)\nfor img_id in tqdm(id_array):\n    maskpath=mask_folder_path+\"/mask_rle_\"+img_id+\".npy\"\n    if os.path.isfile(maskpath)==False:\n        mask=get_mask(img_id)\n        bboxes_dict[img_id]=get_bboxes(mask)\n        mask= multicell_rle(mask)\n        np.save(maskpath,mask)\nif os.path.exists(\"bboxes.pkl\"):\n    os.remove(\"bboxes.pkl\")\nbboxes_dict_file = open(\"bboxes.pkl\", \"wb\")\npickle.dump(bboxes_dict, bboxes_dict_file)\nbboxes_dict_file.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#visualize that the rle encoded mask numpy file is correct\n#just some random image id\nimport matplotlib.pyplot as plt\ntestmask=load_mask_from_file(\"466a98c6-bbae-11e8-b2ba-ac1f6b6435d0\",\"../input/test-rle-mask\")\nplt.imshow(testmask)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#show cell cropping and resize script here\n#cropped resized cells were saved in the datasets hpa-resized-224x224-cropped-cells-x20\n#example img path:\n#../input/hpa-resized-224x224-cropped-cells-420/00481c70-bba3-11e8-b2b9-ac1f6b6435d0_13.png\n#-->img_id:00481c70-bba3-11e8-b2b9-ac1f6b6435d0\n#-->cell_id:13\n\n#make a dict with all the correct file locations:\n\nimport numpy as np\nfrom tqdm import tqdm\nimport os\nimg_paths={}\nimg_folders = [\"../input/hpa-resized-224x224-cropped-cells-\"+str(i+1)+\"20/\" for i in range(20)]\nfor folder in img_folders:\n    imgs=os.listdir(folder)\n    for img in tqdm(imgs):\n        tmp=img.split(\"_\")\n        tmp=tmp[0]\n        tmp=tmp.split(\"/\")\n        img_id=tmp[-1]\n        tmp=img.split(\"_\")\n        tmp=tmp[-1]\n        tmp=tmp.split(\".png\")\n        cell_id=tmp[0]\n        img_paths[img_id]=folder+img_id+\"_\"+cell_id+\".png\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n#load readymade dataframe with ids, labels, and bounding boxes\n#change ids to new file paths\nimport pickle\nimport pandas as pd\nid_labels_cells_array = pd.read_pickle(\"../input/hpa-bboxes/hpa-data.pkl\")\n\ndef id_to_path(x):\n    return img_paths[x]\n\n#id_labels_cells_array[\"ID\"] = id_labels_cells_array[\"ID\"].apply(id_to_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\nid_labels_cells_array","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\nid_labels_cells_array.to_pickle(\"./hpadataframe_cropped_resized_cells_filepaths.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#load ALL of the info necessary for the training of the network\n#bboxes dict has been saved to dataset hpa-bboxes as a pickle file (bboxesone.pkl)\n#combine all the information and save overall dataframe to hpa-data.pkl\nimport pickle\nfrom tqdm import tqdm\n\nbboxes_dict_file = open(\"../input/hpa-bboxes/bboxesone.pkl\", \"rb\")\nbboxes_dict = pickle.load(bboxes_dict_file)\n\nid_labels_cells_array = pd.DataFrame(columns=['ID','cell','Label','ymin','ymax','xmin','xmax'])\nfor i,img_id in tqdm(enumerate(id_array)):\n    n_cells=len(bboxes_dict[img_id])\n    for j in range(n_cells):\n        id_labels_cells_array = id_labels_cells_array.append({'ID': str(img_id),'cell':str(j+1),'Label':labels_dict[img_id],'ymin':str(bboxes_dict[img_id][j][0]),'ymax':str(bboxes_dict[img_id][j][1]),'xmin':str(bboxes_dict[img_id][j][2]),'xmax':str(bboxes_dict[img_id][j][3])}, ignore_index=True)\n#id_labels_cells_array.to_csv(r'./hpa-dataframe.csv', index = True)\nid_labels_cells_array.to_pickle(\"./hpa-data.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part TWO: Visualization","metadata":{}},{"cell_type":"code","source":"#%%script echo skipping\n#visualize some input images\n\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport numpy as np\n\nimg_id=\"b8f0d89e-bbc0-11e8-b2bb-ac1f6b6435d0\"\nbase=\"../input/hpa-single-cell-image-classification/train/\"\next=\".png\"\n\nr=Image.open(base+img_id+\"_red\"+ext)\ng=Image.open(base+img_id+\"_green\"+ext)\nb=Image.open(base+img_id+\"_blue\"+ext)\ny=Image.open(base+img_id+\"_yellow\"+ext)\n\npaths=[r,g,b,y]\n\nfig, axs = plt.subplots(1, 4, figsize=(20, 10))\nfor path, ax, interp in zip(paths, axs, ['Microtubules', 'Protein of Interest', 'Nucleus', 'Endoplasmatic Reticulum']):\n    ax.imshow(np.asarray(path), vmin=0, vmax=255)\n    ax.set_title(interp)\nplt.show()\n#plt.savefig('image_layers.png')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\n#https://www.kaggle.com/hamditarek/exploring-human-protein-atlas-cell-classification\nlabels_num = [value.split('|') for value in id_labels_array['Label']]\nlabels_num_flat = list(map(int, [item for sublist in labels_num for item in sublist]))\nlabels = [\"\" for _ in range(len(labels_num_flat))]\nfor i in range(len(labels_num_flat)):\n    labels[i] = LABELS[labels_num_flat[i]]\n\nfig, ax = plt.subplots(figsize=(15, 5))\npd.Series(labels).value_counts().plot(kind='barh', fontsize=14)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\narr_len = [len(i) for i in labels_num]\nlengths=set(arr_len)\ncount={}\nfor length in lengths:\n    count[length]=[length,arr_len.count(length)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\ncount=pd.DataFrame.from_dict(count, orient='index',\n                       columns=['labels', 'occurence'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\ncount","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Part THREE: Single-Cell Multi-Label Classification Network  \nhttps://www.kaggle.com/ayuraj/hpa-multi-label-classification-with-tf-and-w-b  \nhttps://cs230.stanford.edu/blog/datapipeline/  \n   ","metadata":{}},{"cell_type":"code","source":"#%%script echo skipping\nimport tensorflow as tf\nprint(tf.__version__)\n\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import *\nimport tensorflow_addons as tfa\n\nimport os\nimport re\nimport cv2\nimport glob\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom functools import partial\nimport matplotlib.pyplot as plt\n\n%matplotlib inline\n\n# Imports for augmentations. \nfrom albumentations import (\n    Compose, RandomCrop, RandomResizedCrop, HorizontalFlip, VerticalFlip, Resize \n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nid_labels_cells_array = pd.read_pickle(\"../input/hpa-cropped-resized-with-labels-and-paths/hpadataframe_cropped_resized_cells_filepaths.pkl\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_labels_cells_array","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n#test the normalization\nimport matplotlib.pyplot as plt\nimport numpy as np\ndef histogram(image):\n    # Display image in top subplot using color map 'gray'\n    plt.subplot(2,1,1)\n    plt.imshow(image, cmap='gray')\n    plt.title('Original image')\n    plt.axis('off')\n\n    # Flatten the image into 1 dimension: pixels\n    pixels = image.flatten()\n\n    # Display a histogram of the pixels in the bottom subplot\n    plt.subplot(2,1,2)\n    pdf = plt.hist(pixels, bins=64, range=(0,1), density=True,\n                   color='red', alpha=0.4)\n    plt.grid('off')\n\n    # Use plt.twinx() to overlay the CDF in the bottom subplot\n    plt.twinx()\n\n    # Display a cumulative histogram of the pixels\n    cdf = plt.hist(pixels, bins=64, range=(0,1),\n                   density=True, cumulative=True,\n                   color='blue', alpha=0.4)\n\n    # Specify x-axis range, hide axes, add title and display plot\n    plt.xlim((0,1))\n    plt.grid('off')\n    plt.title('PDF & CDF (original image)')\n    plt.show()\n\ndef numpyhisto(img):\n    hist,bins = np.histogram(img.flatten(),256,[0,1])\n    cdf = hist.cumsum()\n    cdf_normalized = cdf * float(hist.max()) / cdf.max()\n    plt.plot(cdf_normalized, color = 'b')\n    plt.hist(img.flatten(),256,[0,1], color = 'r')\n    plt.xlim([0,1])\n    plt.legend(('cdf','histogram'), loc = 'upper left')\n    plt.show()\n\n# Load the image into an array: image\nimage = plt.imread('../input/hpa-composite-images-1-of-20/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0.png')\nhistogram(image)\nnumpyhisto(image)\nrgb = tf.io.read_file(\"../input/hpa-composite-images-1-of-20/000a6c98-bb9b-11e8-b2b9-ac1f6b6435d0.png\")\nimage = tf.image.decode_png(rgb, channels=3)\nimage=tf.image.per_image_standardization(image)\nimage=image.numpy()\nhistogram(image)\nnumpyhisto(image)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n#WORKING_DIR_PATH = '../input/hpa-single-cell-image-classification/'\n\nIMG_WIDTH = 224\nIMG_HEIGHT = 224\nBATCH_SIZE = 16\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n#shuffle dataframe to prevent overfitting\n#from sklearn.utils import shuffle\n#id_labels_cells_array = shuffle(id_labels_cells_array)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n#set an amount of folds to split dataframe into --> k-fold cross validation\n# explanation: https://towardsdatascience.com/cross-validation-explained-evaluating-estimator-performance-e51e5430ff85\nN_FOLDS=5\n#choose which one of the 5 folds will be used as validation set this time\ni_VAL_FOLD=1\nid_labels_cells_array=np.array_split(id_labels_cells_array, N_FOLDS+1) #add one extra part for testing set\ndf_test_split=id_labels_cells_array[-1]\nid_labels_cells_array=id_labels_cells_array[:-1]\ni_training = [i for i in range(N_FOLDS)]\ni_training.pop(i_VAL_FOLD-1)\ni_validation=i_VAL_FOLD-1\ndf_train_split=list()\nfor i in i_training:\n    df_train_split.append(id_labels_cells_array[i])\ndf_train_split=pd.concat(df_train_split)\ndf_val_split=id_labels_cells_array[i_validation]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n#analyze class imbalance and set up class weights here\n#https://www.analyticsvidhya.com/blog/2020/10/improve-class-imbalance-class-weights/\ny_train=df_train_split[\"Label\"].apply(lambda x:list(map(int, x.split(\"|\"))))\ny_train=y_train.values\ny_train=np.concatenate(y_train)\nfrom sklearn.utils import class_weight\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(y_train),\n                                                 y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\ntmp_dict={}\nfor i in range(len(LABELS)):\n    tmp_dict[i]=class_weights[i]\nclass_weights=tmp_dict\nclass_weights","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\nLABELS","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n@tf.function\ndef multiple_one_hot(cat_tensor, depth_list):\n    \"\"\"Creates one-hot-encodings for multiple categorical attributes and\n    concatenates the resulting encodings\n\n    Args:\n        cat_tensor (tf.Tensor): tensor with mutiple columns containing categorical features\n        depth_list (list): list of the no. of values (depth) for each categorical\n\n    Returns:\n        one_hot_enc_tensor (tf.Tensor): concatenated one-hot-encodings of cat_tensor\n    \"\"\"\n    one_hot_enc_tensor = tf.one_hot(cat_int_tensor[:,0], depth_list[0], axis=1)\n    for col in range(1, len(depth_list)):\n        add = tf.one_hot(cat_int_tensor[:,col], depth_list[col], axis=1)\n        one_hot_enc_tensor = tf.concat([one_hot_enc_tensor, add], axis=1)\n\n    return one_hot_enc_tensor\n\ndef resize_val_image(image, label):\n    return tf.image.resize(image, [IMG_HEIGHT, IMG_WIDTH]), label\ndef resize_train_image(image, label):\n    return tf.image.resize(image, [IMG_HEIGHT, IMG_WIDTH]), label\n\n@tf.function\ndef load_image(df_dict):\n    # Load image\n    rgb = tf.io.read_file(df_dict['ID'])\n    image = tf.image.decode_png(rgb, channels=3)\n    #image = tf.image.crop_to_bounding_box(image, int(df_dict['ymin']), int(df_dict['xmin']), int(df_dict['ymax']) - int(df_dict['ymin']), int(df_dict['xmax']) - int(df_dict['xmin']))\n    #image = tf.image.resize(image, [IMG_HEIGHT, IMG_WIDTH])\n    #https://medium.com/@kyawsawhtoon/a-tutorial-to-histogram-equalization-497600f270e2\n    #image=tf.image.per_image_standardization(image)\n    \n    # Parse label\n    label = tf.strings.split(df_dict['Label'], sep='|')\n    label = tf.strings.to_number(label, out_type=tf.int32)\n    label = tf.reduce_sum(tf.one_hot(indices=label, depth=19), axis=0)\n    \n    return image, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n# Consume training CSV \ntrain_ds = tf.data.Dataset.from_tensor_slices(dict(df_train_split))\nval_ds = tf.data.Dataset.from_tensor_slices(dict(df_val_split))\n\n# Training Dataset\ntrain_ds = (\n    train_ds\n    .shuffle(1024)\n    .map(load_image, num_parallel_calls=AUTOTUNE)\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.experimental.AUTOTUNE)\n)\n\n# Validation Dataset\nval_ds = (\n    val_ds\n    .shuffle(1024)\n    .map(load_image, num_parallel_calls=AUTOTUNE)\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.experimental.AUTOTUNE)\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\ndef get_label_name(labels):\n    l = np.where(labels == 1.)[0]\n    label_names = []\n    for label in l:\n        label_names.append(LABELS[label])\n        \n    return '-'.join(str(label_name) for label_name in label_names)\n\ndef show_batch(image_batch, label_batch):\n  plt.figure(figsize=(20,20))\n  for n in range(10):\n      ax = plt.subplot(5,5,n+1)\n      plt.imshow(image_batch[n])\n      plt.title(get_label_name(label_batch[n].numpy()))\n      plt.axis('off')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n# Training batch\nimage_batch, label_batch = next(iter(train_ds))\nshow_batch(image_batch, label_batch)\n#print(label_batch)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n\ndef get_model():\n    base_model = tf.keras.applications.EfficientNetB0(include_top=False, weights='imagenet')\n    base_model.trainable = True\n\n    inputs = Input((IMG_HEIGHT, IMG_WIDTH, 3))\n    x = base_model(inputs, training=True)\n    x = GlobalAveragePooling2D()(x)\n    outputs = Dense(len(LABELS), activation='sigmoid')(x)\n    \n    return Model(inputs, outputs)\n\ntf.keras.backend.clear_session()\nmodel = get_model()\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\nearlystopper = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss', patience=10, verbose=0, mode='min',\n    restore_best_weights=True\n)\n\nlronplateau = tf.keras.callbacks.ReduceLROnPlateau(\n    monitor='val_loss', factor=0.5, patience=5, verbose=0,\n    mode='auto', min_delta=0.0001, cooldown=0, min_lr=0\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n#set up checkpoint save\n#source:https://www.tensorflow.org/tutorials/keras/save_and_load\n!pip install -q pyyaml h5py\nimport os\ncheckpoint_path = \"./cp.ckpt\"\ncheckpoint_dir = os.path.dirname(checkpoint_path)\n\n# Create a callback that saves the model's weights\ncp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,\n                                                 save_weights_only=True,\n                                                 verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://stackoverflow.com/questions/43198613/scikit-learn-f1-score-for-list-of-strings\nfrom sklearn.metrics import f1_score\n\ndef f1_weighted(y_true, y_pred):\n    f1=f1_score(binarizer.transform(y_true), \n         binarizer.transform(y_pred), \n         average='weighted')\n    return f1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n# tf.nn.sigmoid_cross_entropy_with_logits used as loss fct\n# tensorflow says it can be used for multi-label multi-class problems\n# https://www.tensorflow.org/api_docs/python/tf/nn/sigmoid_cross_entropy_with_logits\nimport keras.backend as K\nK_epsilon = K.epsilon()\ndef f1(y_true, y_pred):\n    #y_pred = K.round(y_pred)\n    y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), 0.5), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K_epsilon)\n    r = tp / (tp + fn + K_epsilon)\n\n    f1 = 2*p*r / (p+r+K_epsilon)\n    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n    return K.mean(f1)\ndef f1_loss(y_true, y_pred):\n    \n    #y_pred = K.cast(K.greater(K.clip(y_pred, 0, 1), THRESHOLD), K.floatx())\n    tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)\n    tn = K.sum(K.cast((1-y_true)*(1-y_pred), 'float'), axis=0)\n    fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)\n    fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)\n\n    p = tp / (tp + fp + K_epsilon)\n    r = tp / (tp + fn + K_epsilon)\n\n    f1 = 2*p*r / (p+r+K_epsilon)\n    f1 = tf.where(tf.math.is_nan(f1), tf.zeros_like(f1), f1)\n    return 1-K.mean(f1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\nimport tensorflow as tf\nimport timeit\n\ndevice_name = tf.test.gpu_device_name()\nif \"GPU\" not in device_name:\n    print(\"GPU device not found\")\nprint('Found GPU at: {}'.format(device_name))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n# Initialize model\ntf.keras.backend.clear_session()\nmodel = get_model()\n\n#model.load_weights(checkpoint_path)\n\n# Compile model\nmodel.compile(optimizer='adam', loss=f1_loss, metrics=f1)\n\n# Train\nhistory=model.fit(train_ds,\n                  epochs=20,\n                  validation_data=val_ds,\n                  class_weight=class_weights,\n                  callbacks=[cp_callback,earlystopper])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\nhistory.history","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n#source: https://machinelearningmastery.com/display-deep-learning-model-training-history-in-keras/\n# list all data in history\nprint(history.history.keys())\n# summarize history for f1\nplt.plot(history.history['f1'])\nplt.plot(history.history['val_f1'])\nplt.title('model f1 score')\nplt.ylabel('f1 score')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## confusion matrix","metadata":{}},{"cell_type":"code","source":"#%%script echo skipping\nfor element in train_ds.as_numpy_iterator():\n    x=element[0]\n    y_true=element[1]\n    break\ny_pred = model.predict(x)\nprint(x.shape)\nprint(y_true.shape)\nprint(y_pred.shape)\n#print(\"X:\")\n#print(x)\nprint(\"y_true:\")\nprint(y_true)\nprint(\"y_pred:\")\nprint(y_pred)\n#y_pred=np.argmax(y_pred,axis=1)\ny_pred[y_pred>0.5] = 1\ny_pred[y_pred<0.5] = 0\nprint(\"y_pred:\")\nprint(y_pred)\n#y_pred=np.eye(len(LABELS))[y_pred]\n#print(\"y_pred:\")\n#print(y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(y_true, y_pred, average='weighted')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%script echo skipping\n#https://scikit-learn.org/stable/modules/generated/sklearn.metrics.multilabel_confusion_matrix.html\nimport sklearn.metrics as skm\n\ncm = skm.multilabel_confusion_matrix(y_true, y_pred)\nprint(cm)\nprint(skm.classification_report(y_true,y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\ntarget_names = np.array(list(LABELS.values()))\nprint(target_names)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%script echo skipping\nimport sklearn.metrics as skm\ncm = skm.multilabel_confusion_matrix(y_true, y_pred)\nprint(cm)\nprint( skm.classification_report(y_true,y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mlxtend.evaluate import confusion_matrix\n\ny_target =    [1, 1, 1, 0, 0, 2, 0, 3]\ny_predicted = [1, 0, 1, 0, 0, 2, 1, 3]\n\ncm = confusion_matrix(y_target=y_target, \n                      y_predicted=y_predicted, \n                      binary=False)\ncm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}