{"cells":[{"metadata":{"trusted":true,"_uuid":"9c8842beab8a76f2dc284d0f14450d59bb3de644"},"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nimport sys\nimport cv2\nimport gc\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d54496a0f4918ae08b8009235cbd0267270a873d"},"cell_type":"code","source":"root_dir=\"../input\"#r\"C:\\my_projects\\protein_identification\"\ntrain_dir=os.path.join(root_dir, \"train\")\ntest_dir=os.path.join(root_dir, \"test\")\ntrain_csv_path=os.path.join(root_dir,\"train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dffec2497ce6c57426964aacce9fa7b57651e3c1"},"cell_type":"code","source":"ORIG_IMAGE_SIZE=512","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e71753cee3db7da12d932a0e364b4299a6e7242"},"cell_type":"code","source":"# read the training csv\ntrain_csv = pd.read_csv(train_csv_path)\nprint(train_csv.shape)\ntrain_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a479e313f0ceef397c3c8e15d2f2c1694830bec8"},"cell_type":"code","source":"class_names = {\n    0:\"Nucleoplasm\",\n    1:\"Nuclear membrane\", \n    2:\"Nucleoli\", \n    3:\"Nucleoli fibrillar center\", \n    4:\"Nuclear speckles\", \n    5:\"Nuclear bodies\", \n    6:\"Endoplasmic reticulum\", \n    7:\"Golgi apparatus\", \n    8:\"Peroxisomes\", \n    9:\"Endosomes\", \n    10:\"Lysosomes\", \n    11:\"Intermediate filaments\", \n    12:\"Actin filaments\", \n    13:\"Focal adhesion sites\", \n    14:\"Microtubules\", \n    15:\"Microtubule ends\", \n    16:\"Cytokinetic bridge\", \n    17:\"Mitotic spindle\", \n    18:\"Microtubule organizing center\", \n    19:\"Centrosome\", \n    20:\"Lipid droplets\", \n    21:\"Plasma membrane\", \n    22:\"Cell junctions\", \n    23:\"Mitochondria\", \n    24:\"Aggresome\", \n    25:\"Cytosol\", \n    26:\"Cytoplasmic bodies\", \n    27:\"Rods & rings\" \n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1bd7027814814ba8bb72767e916aad4e8b9754fb"},"cell_type":"markdown","source":"## Let's take a look at class stats"},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"dd039f820b959b4e44959b7e7c58596796af9ffc"},"cell_type":"code","source":"# split the targets in train csv\n\ndef split_classes(row):\n    for cls_num in row[\"Target\"].split():\n        train_csv.loc[row.name, class_names[int(cls_num)]]=1\n\nfor cls_num, cls_name in class_names.items():\n    train_csv[cls_name]=0\n\n# train_csv[\"splitted\"] = train_csv[\"Target\"].apply(lambda x: i+1 for i in x.split())\ntrain_csv.apply(split_classes, axis=1)\ntrain_csv.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"810c5647f95d9810ddbc22189fcb2dff9459068f"},"cell_type":"code","source":"counts=train_csv[list(class_names.values())].sum().sort_values(ascending=False)\ncounts = counts.to_frame(\"cnts\")\nplt.figure(figsize=(20, 10))\nsns.barplot(counts.index, counts.cnts)\nplt.xticks(rotation=70)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82a2a5ab47d59c1f702a899c2a313e2840d6286d"},"cell_type":"code","source":"counts","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd3bd964c8bc3233cb42f5197e81af0cd78baf5f"},"cell_type":"markdown","source":"Based on the above figure the `Nucleoplasm` has the highest samples. Whereas last few labels in the above plot has very low samples. So the dataset is imbalanced. This should be take into account while training a model. "},{"metadata":{"_uuid":"33610b60063ea7656d6182d8c516d283085add8a"},"cell_type":"markdown","source":"### Correlations and co-occurences of Labels\n\nNow we can try to see which labels has highest correlation and co-occurence with each other. Later we will try to see using images to find why it happned? Is it due to the structure of the proteins? "},{"metadata":{"trusted":true,"_uuid":"3840fe87646e12bd2c0b7d0201cab44d06f57284"},"cell_type":"code","source":"corr=train_csv[list(class_names.values())].corr()\nplt.figure(figsize=(7, 7))\nsns.heatmap(corr, linewidths=0.05, linecolor='b', square=True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe7a1b6f66d667ff548450a73727ca10e9ecd185"},"cell_type":"markdown","source":"In the above diagram it looks like the `Endosomes` and the `Lysosomes` are highly correlated. This is not further suprising since they look very similiar. Plus by looking into the train_csv Lysosomes are always occur with the Endosomes. There's no single Lysosomes record. There's also some correlation between `Cytokinetic bridge` and `Mitotic spindle`."},{"metadata":{"trusted":true,"_uuid":"027138e9b878c304f05f4441dca6b38e8fcaf1aa"},"cell_type":"code","source":"#for labels in train_csv.values():\n\nco_occur_map = pd.DataFrame(index=list(class_names.values()), columns=list(class_names.values()))\nco_occur_map=co_occur_map.fillna(0)\n\ndef find_create_map(row):\n    classes = row.split()\n    for r in classes:\n        for c in classes:\n            co_occur_map.loc[class_names[int(r)], class_names[int(c)]] += 1\n\ntrain_csv[\"Target\"].apply(find_create_map)\nco_occur_map.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab13afe40978278e2fe89c785f5cff887afb27a1"},"cell_type":"code","source":"# making 0s to same classes so the map can be seen well\nfor cls in class_names.values():\n    co_occur_map.loc[cls, cls]=0\n\nplt.figure(figsize=(7, 7))\nsns.heatmap(co_occur_map, cmap=\"jet\", linewidths=0.05, linecolor='b', square=True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a14d98d5a72523663ade8809ded635b949dbfd9"},"cell_type":"markdown","source":"Based on the above diagram we can see that few of the classes has high co-occurence. Most of them are with the `Nucleoplasm`. Probably because occurence of that is more than any other classes. We'll try to see it in images and find out what's the similarity."},{"metadata":{"trusted":true,"_uuid":"229fb08e03566b28985e436e1768bf2cbcbed492"},"cell_type":"code","source":"# If we remove the the Nucleoplasm then see which classes has more co-occurence.\nco_map = co_occur_map.drop(index=[\"Nucleoplasm\"])\nco_map = co_map.drop(columns=[\"Nucleoplasm\"])\n\nplt.figure(figsize=(7, 7))\nsns.heatmap(co_map, cmap=\"jet\", linewidths=0.05, linecolor='b', square=True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3bc7ff375dc1766e2bc388fdf12f521dc2599ca7"},"cell_type":"markdown","source":"By looking at above figure we can see that the second highest class `Cytosol` has more co-occurence than other but it also reveals more co-occurence with some other classes like `nucleoli`, `Plasma membrance`, `Golgi apparatus` etc. All of these classes are in top 10 classes. So based on that we can conclude that it is very common to have co-occurences in this data. **So in our model we may need to emphasize on predicting multiple classes**"},{"metadata":{"_uuid":"f46806fa15da9b4ca7d9a76cae40df8efe4f64a2"},"cell_type":"markdown","source":"### EDA based on the images\n\nLet's check the images for different classes, apply some image processing and try to find out some above questions in order to get some more understanding of the data. Which can help to build a good model."},{"metadata":{"trusted":true,"_uuid":"15d4222edf260899ca6327ac4d1adf5eee814fa7"},"cell_type":"code","source":"# show the random protein images\n\nnum_random = 4\n\ncolors = [ \"_green.png\", \"_blue.png\", \"_red.png\", \"_yellow.png\"]\ncmaps =[\"Greens\", \"Blues\", \"Reds\", \"YlOrBr\"]\n\nrnd_imgs = random.sample(train_csv[\"Id\"].values.tolist(), num_random)\n\nfor img in rnd_imgs:\n    plt.figure(figsize=(20, 10))\n    \n    green_img_name = []\n    classes = train_csv[train_csv[\"Id\"]==img][\"Target\"].values.tolist()\n    for cls in  classes[0].split():\n        green_img_name.append(class_names[int(cls)])\n    \n    green_img_name = \",\".join(green_img_name)\n      \n    \n    for j, color in enumerate(colors):\n        plt.subplot(1, 4, j+1)\n        if j == 0:\n            plt.title(green_img_name)\n        if j == 1:\n            plt.title(\"Nucleus\")\n        if j == 2:\n            plt.title(\"Microtubules\")\n        if j == 3:\n            plt.title(\"ER\")\n        plt.grid(False)\n        image = cv2.imread(os.path.join(train_dir,img+color), 0)\n        image.astype(float)\n        plt.imshow(image, cmap=cmaps[j])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f2306b7491f46f8a5255c8d469dcaadded267221"},"cell_type":"markdown","source":" **Lets further take a quick look at an overlay of the channels**"},{"metadata":{"trusted":true,"_uuid":"d31b80f053e7ea8dc3a30d1ee2575f34810f14ce"},"cell_type":"code","source":"def display_combined_rgb_img(img):\n    print(os.path.join(train_dir,img))\n    fig, ax = plt.subplots(1, 1, figsize=(10, 10))\n\n    all_images = np.empty((512,512,4))\n    for i, color in enumerate(['red', 'green', 'yellow', 'blue']):\n        all_images[:,:,i] = plt.imread(os.path.join(train_dir,img+\"_{}.png\").format(color))\n\n    # define transformation matrix\n    # note that yellow is made usign red and green\n    # but you can tune this color conversion yourself\n    T = np.array([[1,0,1,0],[0,1,1,0],[0,0,0,1]])\n\n    # convert to rgb\n    rgb_image = np.matmul(all_images.reshape(-1, 4), np.transpose(T))\n    rgb_image = rgb_image.reshape(all_images.shape[0], all_images.shape[0], 3)\n    rgb_image = np.clip(rgb_image, 0, 1)\n\n    # plot\n    ax.imshow(rgb_image)\n    ax.set(xticks=[], yticks=[])\n\n    return rgb_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26fe7be4c73fec27e35ebd0edbc52373e75809e2"},"cell_type":"code","source":"img=random.sample(train_csv.Id.values.tolist(), 1)[0]\ndisplay_combined_rgb_img(img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1215bc00250c5215420ec746ff0236e8560edb84"},"cell_type":"code","source":"T = np.array([[1,0,1,0],[0,1,1,0],[0,0,0,1]])\nnp.transpose(T)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"5569f7211c23a7b884a0208af01fa350b9668608"},"cell_type":"code","source":"# let's see random image of Nucleoplasm and Cytosol to understand the co-occurence between those classes\nfig_size=(1,1)\n\n# get first image where there's only Nucleoplasm\nnue=train_csv[train_csv.Target.isin(['0'])]\nnue=nue.reset_index(drop=True).iloc[0][\"Id\"]\ndisplay_combined_rgb_img(nue)\nplt.title(\"Nucleoplasm\")\nplt.figure(figsize=fig_size)\nplt.show()\n\n# get first image where there's only Cytosol\ncyt=train_csv[train_csv.Target.isin(['25'])]\ncyt=cyt.reset_index(drop=True).iloc[0][\"Id\"]\nplt.figure(figsize=fig_size)\ndisplay_combined_rgb_img(cyt)\nplt.title(\"Cytosol\")\nplt.show()\n\n# get first image where there's both Nucleoplasm and Cytosol together\nnue_cyt=train_csv.query('Nucleoplasm==1 & Cytosol==1')\nnue_cyt=nue_cyt.reset_index(drop=True).iloc[1][\"Id\"]\nplt.figure(figsize=fig_size)\ndisplay_combined_rgb_img(nue_cyt)\nplt.title(\"Nucleoplasm & Cytosol\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0706df38b76f261fcd84c5219f52043c97b46658"},"cell_type":"markdown","source":"By looking at above images, it seems like it will be hard to make any visual connection just by looking at the images. Also,this [Kaggle question](https://www.kaggle.com/c/human-protein-atlas-image-classification/discussion/68597#414634) indicates that no cell will show the similar pattern as another. Probably it requires domain knowledge to derive this type of conclusion. I don't have any domain knowledge in this area so I will leave the co-occurence topic here. I will try to come back if I find out some quick explnation."},{"metadata":{"_uuid":"278d585bcf924bb3eb72f8de754acc2c9237ec8f"},"cell_type":"markdown","source":"### Cell Segmentation\n\nBy performaing the simple cell Segmentation it can give insight that what we should look at when developing the model to  classify the images.\n\nAlso, By looking at images above we saw that the amount and size of cells varies a lot and therefore the POI shows a great deal of variance. By using the simple segmentation it can be studied for the various types of image. Which can helpful to build the final model.\n\nFor the segmentation I took the reference of [this](https://www.kaggle.com/weegee/protein-where-art-thou) notebook"},{"metadata":{"trusted":true,"_uuid":"5f17b6c837267bc4837ebc9df7249bd7569fc341"},"cell_type":"code","source":"img=random.sample(train_csv.Id.values.tolist(), 1)[0]\nrgb_image=display_combined_rgb_img(img)\n#print(rgb_image.shape)\nrgb_image=np.uint16(rgb_image)\ngray_img=cv2.cvtColor(rgb_image, cv2.COLOR_RGB2GRAY) # converting due to thresholding should be applied on gray.\n\n\nplt.title(\"Original Image\")\nplt.imshow(gray_img, cmap=\"gray\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8460f87dae1715d646b3296cea8c20987f671737"},"cell_type":"markdown","source":"Using above method the image doesn't show any details, so let's try another strategy. "},{"metadata":{"trusted":true,"_uuid":"5c6ac5a35d865d177042217f9e67bd62543168c4"},"cell_type":"code","source":"img=random.sample(train_csv.Id.values.tolist(), 1)[0]\n\npoi = cv2.imread(os.path.join(train_dir,img+\"_green.png\"), 0)\nnuc = cv2.imread(os.path.join(train_dir,img+\"_blue.png\"), 0)\nmt = cv2.imread(os.path.join(train_dir,img+\"_yellow.png\"), 0)\ner = cv2.imread(os.path.join(train_dir,img+\"_red.png\"), 0)\ncomposit = cv2.add(poi,nuc, mt, er)\n#composit = cv2.resize(composit, (256,256))\n\nplt.figure()\nplt.grid(False)\nplt.title(\"original\")\nplt.imshow(composit, cmap=\"gray\")\nplt.show()\n\n\nret,thresh1 = cv2.threshold(composit,0,255,cv2.THRESH_BINARY)\nret,thresh2 = cv2.threshold(composit,0,255,cv2.THRESH_BINARY_INV)\nret,thresh3 = cv2.threshold(composit,0,255,cv2.THRESH_TRUNC)\nret,thresh4 = cv2.threshold(composit,0,255,cv2.THRESH_TOZERO)\nret,thresh5 = cv2.threshold(composit,0,255,cv2.THRESH_TOZERO_INV)\nret,thresh6 = cv2.threshold(composit,0,255,cv2.THRESH_OTSU)\n\nplt.figure(figsize=(10,10))\ntitles = ['BINARY','BINARY_INV','TRUNC','TOZERO','TOZERO_INV', 'THRESH_OTSU']\nimages = [thresh1, thresh2, thresh3, thresh4, thresh5, thresh6]\nfor i in range(6):\n    plt.subplot(2,3,i+1),plt.imshow(images[i],'gray')\n    plt.title(titles[i])\n    plt.xticks([]),plt.yticks([])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a7a34e6cc8ad4f393729f28db7bb04c57fee7728"},"cell_type":"markdown","source":"In all of above thrsholds the THRESH_OTSU works better than any other thresholds, so we'll use it for the further processing."},{"metadata":{"trusted":true,"_uuid":"e7a0f60ff5350673fa103e0e041b2afbabd009f2"},"cell_type":"code","source":"nuc = cv2.imread(os.path.join(train_dir,img+\"_green.png\"), 0)\n#nuc = cv2.resize(nuc, (256,256))\nplt.figure(figsize=(15,5))\n\nplt.subplot(131)\nplt.grid(False)\nplt.title(\"Protein of Interest staining\")\nplt.imshow(nuc, cmap='gray')\n\nt, thresh = cv2.threshold(nuc, 0,255,cv2.THRESH_OTSU)\n\nplt.subplot(132)\nplt.grid(False)\nplt.title(\"OTSU thresholding\\n of POI staining\")\nplt.imshow(thresh, cmap=\"gray\")\n\nkernel = np.ones((6,6),np.uint8)\nclosing = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)\n\nplt.subplot(133)\nplt.grid(False)\nplt.title(\"OTSU thresholding\\nafter closing operation\")\nplt.imshow(closing, cmap=\"gray\")\n\nim, contours,hierarchy = cv2.findContours(closing, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n\nt=cv2.drawContours(composit, contours, -1, (0,0,0), 2)\n\nplt.figure(figsize=(10,10))\nplt.grid(False)\nplt.title(\"POI contours drawn in composit image\")\nplt.imshow(t)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43ea913d23ac022b6587f18f82d81e7f14e52fc1"},"cell_type":"markdown","source":"By looking at the image above it shows some intersting structures. Those may be the protein of interest or some other particals. Based on this we can derieve that when the modeling will done it should look at this type of similar structure and in the inference it should point this type structures. Which matches the goal of this challange to do the localization of the samples."},{"metadata":{"_uuid":"7d83bc390347a59fdc4656b3ab3b25bf8c9ab20a"},"cell_type":"markdown","source":"## Modeling\n\nBased on the description, the model shoule predict protein localization labels for each sample. There are in total 28 different labels present in the dataset. And as we saw before there can be multiple labels per image. So it is a multi label problem. In addition to that the data is imabalanced training data. So we need to find out a technique where the imabalanced classes don't affect in inference phase. \n\nAccording to description of the problem, the green filter should be used to predict the label, and the other filters are used as references. So below, we'll try to create model using the green filter images and we'll see whether the reference images helps or not.\n\nHere, we'll try basic CNNs and an advanced algorithms."},{"metadata":{"trusted":true,"_uuid":"2e7abc2c36d021ccaff0b8f866cdd8152d2269b2"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b1b3c8a67bb91bdfaa90f397e620c651abfca74"},"cell_type":"markdown","source":"### Create the dataset"},{"metadata":{"trusted":true,"_uuid":"4a9df826dc79aaea68cfb997f2cc01b18cb2513f"},"cell_type":"code","source":"# Load the green images in train and test\n\ngc.collect()\n\nMAX_IMG_FOR_MODELING = 7000 # choosing only this much images to avoid memory error \nshuffle(train_csv)\nsubset_data=train_csv[:MAX_IMG_FOR_MODELING]\n\ntrain_imgs, test_valid_imgs, train_labels, test_valid_labels = train_test_split(subset_data[\"Id\"], \n                                                                    subset_data[list(class_names.values())], \n                                                                    test_size=0.4)\n\nvalid_imgs, test_imgs, valid_labels, test_labels = train_test_split(test_valid_imgs, \n                                                                    test_valid_labels, \n                                                                    test_size=0.5)\n\ntrain_labels = train_labels.values\nvalid_labels = valid_labels.values\ntest_labels = test_labels.values\n\nprint(train_imgs.shape)\nprint(valid_imgs.shape)\nprint(test_imgs.shape)\n\ndef load_images(img_array):\n    images = []\n    for i in img_array:\n        image = cv2.imread(os.path.join(train_dir, i+\"_green.png\"))\n        image.resize(ORIG_IMAGE_SIZE, ORIG_IMAGE_SIZE, 1)\n        images.append(image.astype(np.float32))\n\n    images=np.array(images)\n    return images\n\ntrain_images = load_images(train_imgs)\nprint(train_images.shape)\nvalid_images = load_images(valid_imgs)\nprint(valid_images.shape)\ntest_images = load_images(test_imgs)\nprint(test_images.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"978599166eb5300ad79bee833f1e9923345d5a69"},"cell_type":"markdown","source":"#### A very simple CNN using TF"},{"metadata":{"trusted":true,"_uuid":"dd8bd64e009ff93807b8f85c2a0ae508898032e0"},"cell_type":"code","source":"batch_size=16\npatch_size=3 #5\ndepth=1\nnum_hidden=64\nnum_channels=1 # grayscale\nimage_size=ORIG_IMAGE_SIZE\nnum_labels=len(list(class_names.values()))\nnum_steps = 20 #50 #100","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6baecd00e70124d526fabdd10c962e0577115062"},"cell_type":"markdown","source":"### Using the Keras library\n\nRun a simple sequential model using Keras"},{"metadata":{"trusted":true,"_uuid":"3a37ea1c2acf644e3a34b66b1cfbc88870557bd7"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers import Conv2D, Flatten, Dense, MaxPooling2D , BatchNormalization\nfrom keras.models import Sequential\nfrom keras.optimizers import RMSprop, Adam\nfrom keras import regularizers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4953d3e0df83a8fe126a4165bef27fe88834d22f"},"cell_type":"code","source":"data_gen = ImageDataGenerator()\ndata_gen.fit(train_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fb8296b9a841c7c7e2256f6b32acd547f15b541"},"cell_type":"code","source":"l2_rate = 0.5\nmodel = Sequential()\nmodel.add(Conv2D(32, (patch_size, patch_size), activation='relu', input_shape=(ORIG_IMAGE_SIZE, ORIG_IMAGE_SIZE, 1), \n                 strides=(1,1), padding=\"same\", kernel_regularizer=regularizers.l2(0.01)))\nmodel.add(BatchNormalization())\n# model.add(Conv2D(64, (patch_size, patch_size), activation='relu', strides=(1,1), padding=\"same\", kernel_regularizer=regularizers.l2(l2_rate)))\nmodel.add(Conv2D(64, (patch_size, patch_size), activation='relu', strides=(1,1), padding=\"same\", kernel_regularizer=regularizers.l2(l2_rate)))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Conv2D(128, (patch_size, patch_size), activation='relu', strides=(1,1), padding=\"same\", kernel_regularizer=regularizers.l2(l2_rate)))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Conv2D(256, (patch_size, patch_size), activation='relu', strides=(1,1), padding=\"same\", kernel_regularizer=regularizers.l2(l2_rate)))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\n# model.add(Conv2D(512, (patch_size, patch_size), activation='relu', strides=(1,1), padding=\"same\", kernel_regularizer=regularizers.l2(l2_rate)))\n# model.add(MaxPooling2D(pool_size=(2, 2)))\n# model.add(Conv2D(1024, (patch_size, patch_size), activation='relu', strides=(1,1), padding=\"same\"))\n# model.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Flatten())\n# model.add(Dense(4096, activation='relu'))\nmodel.add(Dense(len(list(class_names.values())), activation=\"sigmoid\"))\n\nopt = Adam(lr=0.0001)\nmodel.compile(loss=\"binary_crossentropy\", optimizer=opt, metrics=[\"accuracy\"])\n\nmodel.fit_generator(data_gen.flow(train_images, train_labels, batch_size=batch_size), \n                    validation_data=(valid_images, valid_labels),\n                    steps_per_epoch=len(train_images) / batch_size, epochs=num_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75e11d05aeed6f13cd8b5a6551d1f62e96ce204c"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e467f1780725573eda37abb875ec6331d8324441"},"cell_type":"code","source":"# Train vs Validation accuracy and loss\n\ndef loss_over_epochs(model, add_valid=True):\n    hist=model.history.history\n    plt.plot(list(range(num_steps)), hist['loss'], color=\"blue\", label=\"train\")\n    if add_valid:\n        plt.plot(list(range(num_steps)), hist['val_loss'], color=\"orange\", label=\"valid\")\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"loss\")\n    plt.title(\"Losses over the Epochs\")\n\n# accuracy vs epochs\ndef acc_over_epochs(model, add_valid=True):\n    hist=model.history.history\n    plt.plot(list(range(num_steps)), hist[\"acc\"], color=\"blue\", label=\"train\")\n    if add_valid:\n        plt.plot(list(range(num_steps)), hist[\"val_acc\"], color=\"orange\", label=\"valid\")\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(\"accuracy\")\n    plt.title(\"Accuracy over the Epochs\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14d82ac9b98f0454486d41bd4406df00940976ac"},"cell_type":"code","source":"loss_over_epochs(model, add_valid=True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee14dc5448662f47e6067e8c772b880d2ec092ab"},"cell_type":"code","source":"acc_over_epochs(model, add_valid=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63ce4a13a3220d4b7cfabe2a56c91441d45c98b6"},"cell_type":"code","source":"def save_model(model):\n    model_dir=os.path.join(\"models\")\n    if not os.path.exists(model_dir):\n        os.mkdir(model_dir)\n    model.save(os.path.join(model_dir, \"{}_green_img_cnn_reg_stride1.model\".format(MAX_IMG_FOR_MODELING)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fe5dfb3e27246105ff5f8025b708d29033d7265"},"cell_type":"code","source":"score = model.evaluate(test_images, test_labels, batch_size=batch_size)\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"719a3edf02a2eaa6bec87863d75f17e41a474a9e"},"cell_type":"markdown","source":"### Try random image to classify"},{"metadata":{"trusted":true,"_uuid":"bc8ee6643f35b9e175f099cd7b9d6951dc8e30ca"},"cell_type":"code","source":"# Getting image from the test data\n\nrand_img = test_imgs.reset_index(drop=True).sample(1)\nimg_idx = rand_img.index.values.tolist()[0]\nimg_class = test_labels[img_idx]\nclasses=[]\nfor k,v in zip(img_class, list(class_names.values())):\n    if k:\n        classes.append(v)\nprint(classes)\n\nimage=test_images[img_idx]\nimage = image.reshape(1, 512, 512, 1)\n\nproba = model.predict(image)\nproba","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13ab0b1bdf6f9e2c9668a64493875de1252ab6b9"},"cell_type":"code","source":"pred_cls=[]\nfor k,v in zip(proba[0], list(class_names.values())):\n    if k:\n        pred_cls.append(v)\nprint(pred_cls)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"954bd30a1db1a552eb8abfd81c6871304123f7dd"},"cell_type":"markdown","source":"By looking at the accuracy and the loss plots vs epochs, it seems the model is struggling to generalize. If the last 2 cells ran repeatedly most of the time it will give prbablity of the `Neuclioplasm`  higher then any other classes. So it looks like due to unbalanced data it is not genralizing. "},{"metadata":{"_uuid":"9e3d39b7e9a66b74d904ef0bd8fb0436d6a0b13f"},"cell_type":"markdown","source":"### Transfer Learning\n\nBecause of the bad model training and the prediction above the simple models doesn't perform well. Also due to HW limitations I can't train a complex model. So we can try to use the already trained models like VGG, Inception, MobileNet etc."},{"metadata":{"trusted":true,"_uuid":"d1dd883c40eeedc260737b550a759dcf0991380f"},"cell_type":"code","source":"# Most pre-trained models require 3 Channel images. So need to convert grayscale green images to RGB\n\n# def convert_gray_to_rgb(img_set):\n#     rgb_imgs = []\n    \n#     for gray_img in img_set:\n#         rgb_imgs.append(cv2.cvtColor(gray_img, cv2.COLOR_GRAY2RGB))\n    \n#     return np.array(rgb_imgs)\n\n# train_rgbs = convert_gray_to_rgb(train_images)\n# valid_rgbs = convert_gray_to_rgb(valid_images)\n# test_rgbs = convert_gray_to_rgb(test_images)\n\n# print(train_rgbs.shape)\n# print(valid_rgbs.shape)\n# print(test_rgbs.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c32f1ea148de69ea4dfa956e8a2f46d411c7eb88"},"cell_type":"markdown","source":"#### VGG16\n\nLet's try using the VGG16 pre-trained model. And see if the model val and train accuracy is converging or getting improved."},{"metadata":{"trusted":true,"_uuid":"09c279ed89fe94a1e07ad38be0e526628bce7a42"},"cell_type":"code","source":"# from keras.applications.vgg16 import VGG16\n# from keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3783ed1bc4ce307479124b5d96376ee054d73f8","scrolled":true},"cell_type":"code","source":"# # from keras.models import load_model\n# # vgg_model = load_model(\"https://www.kaggle.com/jaccojurg/vgg16-weights-tf/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5\")\n# vgg_model = VGG16(include_top=False, input_shape=(512, 512, 3))\n# vgg_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"20c83a08365c9d9f239cd90664cc30085d9b61d6"},"cell_type":"markdown","source":"Remove the last layer where it predicts 1000 classes. Also, we need to apply sigmoid as the last layer"},{"metadata":{"trusted":true,"_uuid":"c85faf3cc8e84f40623a7c2907aa53fc2d11fadc","scrolled":true},"cell_type":"code","source":"# x = Flatten()(vgg_model.output)\n# x = Dense(4096, activation='relu')(x)\n# x = Dense(4096, activation='relu')(x)\n# predictions = Dense(len(class_names), activation='sigmoid')(x)\n# new_vgg = Model(inputs = vgg_model.input, outputs = predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43a8e20cad44c515a7a05791c3fc7a632a930c54"},"cell_type":"code","source":"# new_vgg.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbf777e90934de0776667929011364c2f350a093"},"cell_type":"code","source":"# # Make the layer weights constant\n# for layer in new_vgg.layers:\n#     layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3dc653cab9a90d8ef2a23a4c2225fccd1fcad95","scrolled":true},"cell_type":"code","source":"# opt = Adam(lr=0.0001)\n# new_vgg.compile(loss=\"binary_crossentropy\", optimizer=opt, metrics=[\"accuracy\"])\n\n# new_vgg.fit_generator(data_gen.flow(train_rgbs, train_labels, batch_size=batch_size),\n#                     validation_data=(valid_rgbs, valid_labels),\n#                     steps_per_epoch=len(train_rgbs) / batch_size, epochs=num_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1ed8d9b81596af8e6ec980d57bb3e6ca512ff34"},"cell_type":"code","source":"# new_vgg.save(\"vgg16_200img.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6baf94f1e6e76d5f4a6a58921212ada1867e4222"},"cell_type":"code","source":"# loss_over_epochs(new_vgg)\n# plt.show()\n# acc_over_epochs(new_vgg)\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4714758e59c8388ec5e7ad4e1eeb309a1df65c4e"},"cell_type":"markdown","source":"By looking at base model and the base use of VGG model it seems the model isn't converging. There isn't much improvements in the training and val accuracy. So I'm continueing more experiments. I'll publish another notebook for that.\n\nAny improvement comments are welcome!"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}