{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":13333,"databundleVersionId":862146,"sourceType":"competition"},{"sourceId":6014291,"sourceType":"datasetVersion","datasetId":3443156}],"dockerImageVersionId":30497,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Table of Contents <a class=\"anchor\"  id=\"contents\"></a>\n* [Imports and Installations](#imports)\n* [Read the Data](#read_data)\n    * [Preaparing Train Dataframe](#prep_train)\n    * [Preaparing Test Dataframe](#prep_test)\n* [Visualizing the Images](#viz_img)\n* [Visualizing the Segmentation Masks](#viz_mask)\n* [Visualizing the Images with Segmentation Masks](#viz_img_mask)\n* [Custom Data Generator](#data_gen)\n* [Train Test Split](#data_split)\n* [Model Definitions](#model_def)\n* [Custom Loss, metric and Optimizer classes](#loss_optim)\n* [Setting up wandb for model logging](#wandb)\n* [Defining Callbacks](#model_callbacks)\n* [Training the Model](#model_train)\n* [Saving and Downloading the trained Model](#model_save)\n* [Evaluate model on Test Set and make submission](#model_eval_test)\n* [Exploring Predicticted Mask on Validation Set](#model_eval_val)\n* [Acknowledgements](#ack)","metadata":{}},{"cell_type":"markdown","source":"# Imports and Installations <a class=\"anchor\"  id=\"imports\"></a>\n[Go back to the Table of Contents](#contents)","metadata":{}},{"cell_type":"code","source":"import os\nimport io\nimport cv2\nimport time\nimport math\nimport random\nimport logging\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n#logging.getLogger(\"tensorflow\").setLevel(logging.WARNING)\n#os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-25T12:53:10.978869Z","iopub.execute_input":"2023-11-25T12:53:10.979505Z","iopub.status.idle":"2023-11-25T12:53:11.140136Z","shell.execute_reply.started":"2023-11-25T12:53:10.979476Z","shell.execute_reply":"2023-11-25T12:53:11.139258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:14.053438Z","iopub.execute_input":"2023-11-25T12:53:14.054084Z","iopub.status.idle":"2023-11-25T12:53:23.236103Z","shell.execute_reply.started":"2023-11-25T12:53:14.054053Z","shell.execute_reply":"2023-11-25T12:53:23.235123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"GPU available!\" if tf.test.is_gpu_available() else \"GPU is not available\")","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:23.237658Z","iopub.execute_input":"2023-11-25T12:53:23.237962Z","iopub.status.idle":"2023-11-25T12:53:25.708554Z","shell.execute_reply.started":"2023-11-25T12:53:23.237936Z","shell.execute_reply":"2023-11-25T12:53:25.707573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntf.random.set_seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:25.709951Z","iopub.execute_input":"2023-11-25T12:53:25.710253Z","iopub.status.idle":"2023-11-25T12:53:25.722811Z","shell.execute_reply.started":"2023-11-25T12:53:25.710225Z","shell.execute_reply":"2023-11-25T12:53:25.722062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\n\nIMG_WIDTH = 2100\nIMG_HEIGHT = 1400\n\n# training params\nR_WIDTH = 576\nR_HEIGHT = 384\nNUM_CHANNELS = 3\nNUM_CLASSES = 4\nNUM_EPOCHS = 20 \nBATCH_SIZE = 4\nTEST_BATCH_SIZE = 32\n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:14:42.808188Z","iopub.execute_input":"2023-11-25T13:14:42.809039Z","iopub.status.idle":"2023-11-25T13:14:44.315671Z","shell.execute_reply.started":"2023-11-25T13:14:42.808996Z","shell.execute_reply":"2023-11-25T13:14:44.314549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:32.086712Z","iopub.execute_input":"2023-11-25T12:53:32.087094Z","iopub.status.idle":"2023-11-25T12:53:32.091841Z","shell.execute_reply.started":"2023-11-25T12:53:32.087062Z","shell.execute_reply":"2023-11-25T12:53:32.090844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- pip install segmentation-models \nhttps://github.com/qubvel/segmentation_models","metadata":{}},{"cell_type":"code","source":"!pip install segmentation-models","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-25T12:53:33.536717Z","iopub.execute_input":"2023-11-25T12:53:33.537539Z","iopub.status.idle":"2023-11-25T12:53:45.839476Z","shell.execute_reply.started":"2023-11-25T12:53:33.537500Z","shell.execute_reply":"2023-11-25T12:53:45.838407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://stackoverflow.com/questions/75433717/module-keras-utils-generic-utils-has-no-attribute-get-custom-objects-when-im","metadata":{}},{"cell_type":"code","source":"import os\nos.environ[\"SM_FRAMEWORK\"] = \"tf.keras\"\n\nfrom tensorflow import keras\nimport segmentation_models as sm","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:45.841574Z","iopub.execute_input":"2023-11-25T12:53:45.841912Z","iopub.status.idle":"2023-11-25T12:53:45.866351Z","shell.execute_reply.started":"2023-11-25T12:53:45.841868Z","shell.execute_reply":"2023-11-25T12:53:45.865474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read the Data <a class=\"anchor\" id=\"read_data\"></a>\n[Go back to the Table of Contents](#contents)","metadata":{}},{"cell_type":"code","source":"data_dir = '../input/understanding_cloud_organization'\ntrain_csv_path = os.path.join(data_dir,'train.csv')\ntest_csv_path = os.path.join(data_dir,\"sample_submission.csv\")\ntrain_image_path = os.path.join(data_dir,'train_images')\ntest_image_path = os.path.join(data_dir,'test_images')","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:45.867533Z","iopub.execute_input":"2023-11-25T12:53:45.867888Z","iopub.status.idle":"2023-11-25T12:53:45.873409Z","shell.execute_reply.started":"2023-11-25T12:53:45.867855Z","shell.execute_reply":"2023-11-25T12:53:45.872382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" ### Preaparing Train Dataframe <a class=\"anchor\" id=\"prep_train\"></a>\n [Go back to the Table of Contents](#contents)","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(train_csv_path).fillna(-1)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:45.875575Z","iopub.execute_input":"2023-11-25T12:53:45.875831Z","iopub.status.idle":"2023-11-25T12:53:50.459109Z","shell.execute_reply.started":"2023-11-25T12:53:45.875808Z","shell.execute_reply":"2023-11-25T12:53:50.458225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Image_Id'] = train_df['Image_Label'].apply(lambda x: x.split('_')[0])\ntrain_df['Label'] = train_df['Image_Label'].apply(lambda x: x.split('_')[1])\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:50.460226Z","iopub.execute_input":"2023-11-25T12:53:50.460485Z","iopub.status.idle":"2023-11-25T12:53:50.496586Z","shell.execute_reply.started":"2023-11-25T12:53:50.460463Z","shell.execute_reply":"2023-11-25T12:53:50.495765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['Label_EncodedPixels'] = train_df.apply(lambda row: (row['Label'], row['EncodedPixels']), axis = 1)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:50.497753Z","iopub.execute_input":"2023-11-25T12:53:50.498367Z","iopub.status.idle":"2023-11-25T12:53:50.830824Z","shell.execute_reply.started":"2023-11-25T12:53:50.498334Z","shell.execute_reply":"2023-11-25T12:53:50.829906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_EncodedPixels = train_df.groupby('Image_Id')['Label_EncodedPixels'].apply(list)\ngrouped_EncodedPixels.head()\ngrouped_EncodedPixels.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:50.831979Z","iopub.execute_input":"2023-11-25T12:53:50.832272Z","iopub.status.idle":"2023-11-25T12:53:51.006033Z","shell.execute_reply.started":"2023-11-25T12:53:50.832247Z","shell.execute_reply":"2023-11-25T12:53:51.005182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = grouped_EncodedPixels.to_frame().reset_index()\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:51.007177Z","iopub.execute_input":"2023-11-25T12:53:51.007522Z","iopub.status.idle":"2023-11-25T12:53:51.026731Z","shell.execute_reply.started":"2023-11-25T12:53:51.007491Z","shell.execute_reply":"2023-11-25T12:53:51.025829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['Fish', 'Flower', 'Gravel', 'Sugar']\n\nfor label in labels:\n    train_df = train_df.assign(**{label: 0})\nfor index, row in train_df.iterrows():\n    for item in row['Label_EncodedPixels']:\n        label, value = item\n        if value == -1:\n            bool_value = 0\n        else:\n            bool_value = 1\n        train_df.loc[index, label] = bool_value\n\ntrain_df['classes'] = train_df.apply(lambda row: [col for col in labels if row[col] == 1], axis=1)\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:51.027733Z","iopub.execute_input":"2023-11-25T12:53:51.028027Z","iopub.status.idle":"2023-11-25T12:53:53.807485Z","shell.execute_reply.started":"2023-11-25T12:53:51.028003Z","shell.execute_reply":"2023-11-25T12:53:53.806585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[labels].sum().plot(kind='bar')\nplt.xlabel('Columns')\nplt.ylabel('Frequency')","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:53.810562Z","iopub.execute_input":"2023-11-25T12:53:53.810854Z","iopub.status.idle":"2023-11-25T12:53:54.097731Z","shell.execute_reply.started":"2023-11-25T12:53:53.810830Z","shell.execute_reply":"2023-11-25T12:53:54.096820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:54.098994Z","iopub.execute_input":"2023-11-25T12:53:54.099341Z","iopub.status.idle":"2023-11-25T12:53:54.116306Z","shell.execute_reply.started":"2023-11-25T12:53:54.099308Z","shell.execute_reply":"2023-11-25T12:53:54.115435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Finding the index of images having all the masks\nfor ix,item in enumerate(train_df['Label_EncodedPixels'][:100]):\n    c1=item[0][-1]!=-1\n    c2=item[1][-1]!=-1\n    c3=item[2][-1]!=-1\n    c4=item[3][-1]!=-1\n    if c1 and c2 and c3 and c4:\n        print(ix)\n        ","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:54.117381Z","iopub.execute_input":"2023-11-25T12:53:54.117695Z","iopub.status.idle":"2023-11-25T12:53:54.129214Z","shell.execute_reply.started":"2023-11-25T12:53:54.117665Z","shell.execute_reply":"2023-11-25T12:53:54.128274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.loc[28]","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:54.130246Z","iopub.execute_input":"2023-11-25T12:53:54.130476Z","iopub.status.idle":"2023-11-25T12:53:54.143653Z","shell.execute_reply.started":"2023-11-25T12:53:54.130456Z","shell.execute_reply":"2023-11-25T12:53:54.142775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ix,item in enumerate(os.listdir(train_image_path)):\n    if item == \"015aa06.jpg\":\n        print(ix)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:54.144747Z","iopub.execute_input":"2023-11-25T12:53:54.145059Z","iopub.status.idle":"2023-11-25T12:53:54.870254Z","shell.execute_reply.started":"2023-11-25T12:53:54.145014Z","shell.execute_reply":"2023-11-25T12:53:54.869304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preaparing Test Dataframe <a class=\"anchor\" id=\"prep_test\"></a>\n[Go back to the Table of Contents](#contents)","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(test_csv_path).fillna(-1)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:54.871604Z","iopub.execute_input":"2023-11-25T12:53:54.872280Z","iopub.status.idle":"2023-11-25T12:53:54.910420Z","shell.execute_reply.started":"2023-11-25T12:53:54.872246Z","shell.execute_reply":"2023-11-25T12:53:54.909571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Image_Id'] = test_df['Image_Label'].apply(lambda x: x.split('_')[0])\ntest_df['Label'] = test_df['Image_Label'].apply(lambda x: x.split('_')[1])\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:54.911428Z","iopub.execute_input":"2023-11-25T12:53:54.911665Z","iopub.status.idle":"2023-11-25T12:53:54.938315Z","shell.execute_reply.started":"2023-11-25T12:53:54.911644Z","shell.execute_reply":"2023-11-25T12:53:54.937368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['Label_EncodedPixels'] = test_df.apply(lambda row: (row['Label'], row['EncodedPixels']), axis = 1)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:54.939380Z","iopub.execute_input":"2023-11-25T12:53:54.939666Z","iopub.status.idle":"2023-11-25T12:53:55.167247Z","shell.execute_reply.started":"2023-11-25T12:53:54.939643Z","shell.execute_reply":"2023-11-25T12:53:55.166305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_EncodedPixels = test_df.groupby('Image_Id')['Label_EncodedPixels'].apply(list)\ngrouped_EncodedPixels.head()\ngrouped_EncodedPixels.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:55.168203Z","iopub.execute_input":"2023-11-25T12:53:55.168466Z","iopub.status.idle":"2023-11-25T12:53:55.289765Z","shell.execute_reply.started":"2023-11-25T12:53:55.168442Z","shell.execute_reply":"2023-11-25T12:53:55.288908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = grouped_EncodedPixels.to_frame().reset_index()\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:55.290841Z","iopub.execute_input":"2023-11-25T12:53:55.291134Z","iopub.status.idle":"2023-11-25T12:53:55.309430Z","shell.execute_reply.started":"2023-11-25T12:53:55.291110Z","shell.execute_reply":"2023-11-25T12:53:55.308555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing the Images  <a class=\"anchor\" id=\"viz_img\"></a>\n[Go back to the Table of Contents](#contents) <br>\n`PIL or CV2 Image (image_width, image_height) ==  Numpy Array (image_height, image_width)`","metadata":{}},{"cell_type":"code","source":"def batchDataLoader(image_dir,img_w= 512, img_h=512, num_channel =4, Batch_Size=32):\n    \n    while True:\n        k=0\n        image_ids = os.listdir(image_dir)\n        num_batches = math.ceil(len(image_ids)/Batch_Size)\n        \n        for batch_no in range(1,num_batches+1): \n            if batch_no < num_batches:\n                batch_size = Batch_Size\n                batch_image_ids = image_ids[k:k+batch_size]\n                image_batch = np.zeros((batch_size, img_h, img_w, num_channel),dtype=np.uint8)\n                for i in range(batch_size):\n                    path = os.path.join(image_dir, image_ids[i])\n                    img = cv2.imread(path)\n                    img =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n                    image_batch[i] = img\n            # for the last batch which could be fractional\n            if batch_no == num_batches:\n                batch_image_ids = image_ids[k:]\n                batch_size = len(batch_image_ids)\n                image_batch = np.zeros((batch_size, img_h, img_w, num_channel),dtype=np.uint8)\n                for i in range(batch_size):\n                    path = os.path.join(image_dir, image_ids[i])\n                    img = cv2.imread(path)\n                    img =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n                    image_batch[i] = img\n            \n            k = k+batch_size\n            print(f\"batch_no = {batch_no}\")\n            yield image_batch","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:55.310509Z","iopub.execute_input":"2023-11-25T12:53:55.310760Z","iopub.status.idle":"2023-11-25T12:53:55.321747Z","shell.execute_reply.started":"2023-11-25T12:53:55.310738Z","shell.execute_reply":"2023-11-25T12:53:55.320781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_width = 2100\nimg_height = 1400\nnum_channel = 3\nBATCH_SIZE = 32\ncurrent_batch = batchDataLoader(train_image_path,img_width,img_height, num_channel, BATCH_SIZE)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:55.322832Z","iopub.execute_input":"2023-11-25T12:53:55.323230Z","iopub.status.idle":"2023-11-25T12:53:55.335519Z","shell.execute_reply.started":"2023-11-25T12:53:55.323194Z","shell.execute_reply":"2023-11-25T12:53:55.334656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = next(current_batch)\nprint(images.shape)\nplt.figure(figsize=(24,8))\nfor i in range(8):\n    ax = plt.subplot(2,4, i+1)\n    plt.imshow(images[i])\n    #plt.title(labels[i])\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:53:55.338345Z","iopub.execute_input":"2023-11-25T12:53:55.338766Z","iopub.status.idle":"2023-11-25T12:54:02.256935Z","shell.execute_reply.started":"2023-11-25T12:53:55.338742Z","shell.execute_reply":"2023-11-25T12:54:02.255752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing the Segmentation Masks <a class=\"anchor\" id=\"viz_mask\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"code","source":"def rle_to_mask(rle_string, height, width):\n    '''\n    convert RLE(run length encoding) string to numpy array\n\n    Parameters: \n    rle_string (str): string of rle encoded mask\n    height (int): height of the mask\n    width (int): width of the mask \n\n    Returns: \n    numpy.array: numpy array of the mask\n    '''\n    \n    rows, cols = height, width\n    \n    if rle_string == -1:\n        return np.zeros((height,width))\n    else:\n        rle_numbers = [int(num_string) for num_string in rle_string.split(' ')]\n        rle_pairs = np.array(rle_numbers).reshape(-1,2)\n        img = np.zeros(rows*cols, dtype=np.uint8)\n        for index, length in rle_pairs:\n            index -= 1\n            img[index:index+length] = 255\n        img = img.reshape(cols,rows)\n        img = img.T\n        img = img/255.0\n        return img\n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:02.258553Z","iopub.execute_input":"2023-11-25T12:54:02.258846Z","iopub.status.idle":"2023-11-25T12:54:02.266621Z","shell.execute_reply.started":"2023-11-25T12:54:02.258820Z","shell.execute_reply":"2023-11-25T12:54:02.265766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_masks_by_img_id(dataframe, image_id):\n    masks = np.zeros((img_height,img_width,4))\n    rle_masks = list(dataframe[dataframe['Image_Id'] == image_id]['Label_EncodedPixels'])[0]\n    fish_mask = rle_to_mask(rle_masks[0][1], img_height, img_width)\n    flower_mask = rle_to_mask(rle_masks[1][1], img_height, img_width)\n    gravel_mask = rle_to_mask(rle_masks[2][1], img_height, img_width)\n    sugar_mask = rle_to_mask(rle_masks[3][1], img_height, img_width)\n    mask_list = [fish_mask,flower_mask,gravel_mask,sugar_mask]\n    for ix, mask in enumerate(mask_list):\n        masks[:,:,ix] = mask\n    return masks","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:02.267667Z","iopub.execute_input":"2023-11-25T12:54:02.267953Z","iopub.status.idle":"2023-11-25T12:54:02.281303Z","shell.execute_reply.started":"2023-11-25T12:54:02.267925Z","shell.execute_reply":"2023-11-25T12:54:02.280377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = '0011165.jpg'\npath = os.path.join(train_image_path,image_id)\nimg = cv2.imread(path)\nimg =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nrle = list(train_df[train_df['Image_Id'] == image_id]['Label_EncodedPixels'])[0][0][1]\n\nm = rle_to_mask(rle,img_height,img_width)\nm = cv2.resize(m, (384,256),interpolation=cv2.INTER_LINEAR)\nm = (m>0).astype(int)\nplt.imshow(m)\nprint(m.shape)\nprint(np.unique(m))\nprint(np.argwhere(m==1)[0])","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:02.751257Z","iopub.execute_input":"2023-11-25T12:54:02.751618Z","iopub.status.idle":"2023-11-25T12:54:03.210510Z","shell.execute_reply.started":"2023-11-25T12:54:02.751591Z","shell.execute_reply":"2023-11-25T12:54:03.209550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = os.listdir(train_image_path)[1]\n#image_id = 'f516a20.jpg'\nmasks = get_masks_by_img_id(train_df, image_id)\nmasks.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:04.706321Z","iopub.execute_input":"2023-11-25T12:54:04.706679Z","iopub.status.idle":"2023-11-25T12:54:04.861048Z","shell.execute_reply.started":"2023-11-25T12:54:04.706650Z","shell.execute_reply":"2023-11-25T12:54:04.860192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = os.listdir(train_image_path)[5390]\n#image_id = 'f516a20.jpg'\nmasks = get_masks_by_img_id(train_df, image_id)\nprint(image_id)\nplt.figure(figsize=(24,4))\nfor ix in range(masks.shape[-1]):\n    ax = plt.subplot(1,4, ix+1)\n    plt.imshow(masks[:,:,ix],cmap=None)\n    plt.axis(\"off\");","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:05.739687Z","iopub.execute_input":"2023-11-25T12:54:05.740426Z","iopub.status.idle":"2023-11-25T12:54:07.291511Z","shell.execute_reply.started":"2023-11-25T12:54:05.740389Z","shell.execute_reply":"2023-11-25T12:54:07.290566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing the Images with Segmentation Masks <a class=\"anchor\" id=\"viz_img_mask\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"code","source":"def draw_label_on_mask(mask, label, obj=plt):\n    '''\n    Function to add labels to the image.\n    '''\n    if np.sum(mask) > 0:\n        y,x = 0,0\n        y,x = np.argwhere(mask==1)[0]\n        y,x = y+50,x+20      \n        obj.text(x,y,label,color='white',)\n    return None","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:10.943665Z","iopub.execute_input":"2023-11-25T12:54:10.944402Z","iopub.status.idle":"2023-11-25T12:54:10.950985Z","shell.execute_reply.started":"2023-11-25T12:54:10.944365Z","shell.execute_reply":"2023-11-25T12:54:10.949995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = os.listdir(train_image_path)[5390]\npath = os.path.join(train_image_path,image_id)\nimg = cv2.imread(path)\nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nimg = cv2.resize(img,(384,256))\nimg = img.astype(np.float32)\nimg = img/255.0\n#img -= img.mean()\n#img /= img.std()\n#standarization changes the color\nprint(img.shape)\nplt.imshow(img);","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:45.550734Z","iopub.execute_input":"2023-11-25T12:54:45.551496Z","iopub.status.idle":"2023-11-25T12:54:45.987590Z","shell.execute_reply.started":"2023-11-25T12:54:45.551462Z","shell.execute_reply":"2023-11-25T12:54:45.986615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = os.listdir(train_image_path)[5390]\npath = os.path.join(train_image_path,image_id)\nimg = cv2.imread(path)\nimg =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nmasks = get_masks_by_img_id(train_df, image_id)\nmask = masks[:,:,1]\nmask = np.clip(mask,0,1)\nmask = np.ma.masked_where(mask == 0, mask)\nplt.imshow(img)\nplt.imshow(mask,alpha=0.7,cmap='PuRd_r')\ndraw_label_on_mask(mask,\"Flower\")\nplt.axis('off');","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:46.270631Z","iopub.execute_input":"2023-11-25T12:54:46.270996Z","iopub.status.idle":"2023-11-25T12:54:47.553644Z","shell.execute_reply.started":"2023-11-25T12:54:46.270968Z","shell.execute_reply":"2023-11-25T12:54:47.552697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = os.listdir(train_image_path)[5390]\npath = os.path.join(train_image_path,image_id)\nimg = cv2.imread(path)\nimg =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nmasks = get_masks_by_img_id(train_df, image_id)\nmask = masks[:,:,1]\nmask = np.clip(mask,0,1)\nmask = np.ma.masked_where(mask == 0, mask)\nbbox = cv2.boundingRect(mask.astype(np.uint8))\ncv2.rectangle(img, bbox, (0, 255, 0), 5)\nplt.imshow(img)\n#plt.imshow(mask,alpha=0.7,cmap='PuRd_r')\ndraw_label_on_mask(mask,\"Flower\")\nplt.axis('off');","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:47.555144Z","iopub.execute_input":"2023-11-25T12:54:47.555443Z","iopub.status.idle":"2023-11-25T12:54:48.853207Z","shell.execute_reply.started":"2023-11-25T12:54:47.555418Z","shell.execute_reply":"2023-11-25T12:54:48.852317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = os.listdir(train_image_path)[5390]\nmasks = get_masks_by_img_id(train_df, image_id)\nmasks = (masks[:,:,0], masks[:,:,1],masks[:,:,2],masks[:,:,3])\n\npath = os.path.join(train_image_path,image_id)\nimg = cv2.imread(path)\nimg =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\ncolormaps = ['PuRd_r', 'Blues_r', 'Purples_r','winter_r'] # colormap_r = inverse colormap\nmask_labels = ['Fish', 'Flower', 'Gravel', 'Sugar']\nplt.figure(figsize=(15,10))\nfor i,(mask,cmap,label)in enumerate(zip(masks,colormaps,mask_labels)):\n    mask = np.clip(mask,0,1)\n    mask = np.ma.masked_where(mask == 0, mask)\n    ax = plt.subplot(2,2, i+1)\n    plt.imshow(img)\n    plt.imshow(mask,alpha=0.7,cmap=cmap) \n    draw_label_on_mask(mask,label)\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:48.854745Z","iopub.execute_input":"2023-11-25T12:54:48.855045Z","iopub.status.idle":"2023-11-25T12:54:53.167416Z","shell.execute_reply.started":"2023-11-25T12:54:48.855013Z","shell.execute_reply":"2023-11-25T12:54:53.166258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = os.listdir(train_image_path)[5390]\nmasks = get_masks_by_img_id(train_df, image_id)\nmasks = (masks[:,:,0], masks[:,:,1],masks[:,:,2],masks[:,:,3])\n\npath = os.path.join(train_image_path,image_id)\nimg = cv2.imread(path)\nimg =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\ncolormaps = ['PuRd_r', 'Blues_r', 'Purples_r','winter_r']\nmask_labels = ['Fish', 'Flower', 'Gravel', 'Sugar']\nplt.figure(figsize=(32,8))\nplt.imshow(img)\nfor i,(mask,cmap,label) in enumerate(zip(masks,colormaps,mask_labels)):\n    mask = np.clip(mask,0,1)\n    mask = np.ma.masked_where(mask == 0, mask)\n    plt.imshow(mask,alpha=0.7,cmap=cmap) # colormap_r = inverse colormap\n    draw_label_on_mask(mask,label)\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:53.169140Z","iopub.execute_input":"2023-11-25T12:54:53.169435Z","iopub.status.idle":"2023-11-25T12:54:56.307232Z","shell.execute_reply.started":"2023-11-25T12:54:53.169409Z","shell.execute_reply":"2023-11-25T12:54:56.306329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport numpy as np\nimport matplotlib\n\ndef show_bounding_boxes(image, mask, labels, colors):\n    \"\"\"Shows the bounding boxes surrounding the polygon in the image, and\n    adds labels to the bounding boxes.\n\n    Args:\n    image: The image.\n    mask: The binary polygon mask.\n    labels: The labels of the objects in the mask.\n    colors: A list of colors to use for the bounding boxes.\n\n    Returns:\n    The image with the bounding boxes and labels drawn on it.\n    \"\"\"\n\n    # Find the bounding boxes of the polygon.\n    bounding_boxes = []\n    for i in range(mask.shape[-1]):\n        bbox = cv2.boundingRect(mask[:, :, i])\n        bounding_boxes.append(bbox)\n\n    # Draw the bounding boxes on the image.\n    for bbox, label, color_name in zip(bounding_boxes, labels, colors):\n        rgb_color = matplotlib.colors.to_rgb(color_name)\n        rgb_color = tuple(value * 255 for value in rgb_color)\n        cv2.rectangle(image, bbox, rgb_color, 10)\n        cv2.putText(image, label, (bbox[0], bbox[1] + 50),\n                    cv2.FONT_HERSHEY_SIMPLEX, 2, (255,255,255), 6)\n\n    return image\n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:56.308688Z","iopub.execute_input":"2023-11-25T12:54:56.308967Z","iopub.status.idle":"2023-11-25T12:54:56.317560Z","shell.execute_reply.started":"2023-11-25T12:54:56.308941Z","shell.execute_reply":"2023-11-25T12:54:56.316405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib\nrgb_color = matplotlib.colors.to_rgb('darkblue')\nrgb_color = tuple(value * 255 for value in rgb_color)\nrgb_color","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:56.319586Z","iopub.execute_input":"2023-11-25T12:54:56.319863Z","iopub.status.idle":"2023-11-25T12:54:56.334988Z","shell.execute_reply.started":"2023-11-25T12:54:56.319839Z","shell.execute_reply":"2023-11-25T12:54:56.334101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = os.listdir(train_image_path)[5390]\nmasks = get_masks_by_img_id(train_df, image_id)\nmasks = masks.astype(np.uint8)\n\n\npath = os.path.join(train_image_path,image_id)\nimg = cv2.imread(path)\nimg =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n\ncolors = ['maroon', 'darkblue', 'purple','teal']\nlabels = ['Fish', 'Flower', 'Gravel', 'Sugar']\n\nimg = show_bounding_boxes(img,masks,labels,colors)\n\nplt.figure(figsize=(32,8))\nplt.imshow(img)\nplt.axis(\"off\");","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:56.336094Z","iopub.execute_input":"2023-11-25T12:54:56.336414Z","iopub.status.idle":"2023-11-25T12:54:57.460192Z","shell.execute_reply.started":"2023-11-25T12:54:56.336384Z","shell.execute_reply":"2023-11-25T12:54:57.459266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_img_with_masks(img,masks,comment=\"\"):\n    \n    colormaps = ['PuRd_r', 'Blues_r', 'Purples_r','winter_r']\n    mask_labels = ['Fish', 'Flower', 'Gravel', 'Sugar']\n    \n    fig, axes = plt.subplots(1,6,figsize=(36,4))\n    axes = axes.ravel()\n    \n    if img.shape[-1]!=3:\n        img_cmap = 'gray'\n    else:\n        img_cmap=None\n        \n    for ix,axis in enumerate(axes):\n        ix = ix%6\n        axis.imshow(img,cmap=img_cmap)\n        axis.axis('off')\n        if ix==0:\n            axis.set_title(\"Main Image\")\n        elif ix==1:\n            for i,(mask,cmap,label) in enumerate(zip(masks,colormaps,mask_labels)):\n                mask = np.clip(mask,0,1)\n                mask = np.ma.masked_where(mask == 0, mask)\n                axis.imshow(mask,alpha=0.7,cmap=cmap)\n                axis.set_title(f\"All the mask {comment}\")\n                draw_label_on_mask(mask,label,axis)\n        elif ix>=2:\n            for i,(mask,cmap,label) in enumerate(zip(masks,colormaps,mask_labels)):\n                mask = np.clip(mask,0,1)\n                mask = np.ma.masked_where(mask == 0, mask)\n                axis = axes[2+i]\n                axis.imshow(mask,alpha=0.4,cmap=cmap)\n                axis.set_title(f\"{label} {comment}\")\n                draw_label_on_mask(mask,label,axis)\n    plt.show()\n    \n    return None","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:57.461615Z","iopub.execute_input":"2023-11-25T12:54:57.462282Z","iopub.status.idle":"2023-11-25T12:54:57.475789Z","shell.execute_reply.started":"2023-11-25T12:54:57.462248Z","shell.execute_reply":"2023-11-25T12:54:57.474963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = os.listdir(train_image_path)[5390]\npath = os.path.join(train_image_path,image_id)\nimg = cv2.imread(path)\nimg =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\nmasks = get_masks_by_img_id(train_df, image_id)\nmasks = (masks[:,:,0], masks[:,:,1],masks[:,:,2],masks[:,:,3])\nshow_img_with_masks(img,masks)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:54:57.476816Z","iopub.execute_input":"2023-11-25T12:54:57.477137Z","iopub.status.idle":"2023-11-25T12:55:06.738358Z","shell.execute_reply.started":"2023-11-25T12:54:57.477103Z","shell.execute_reply":"2023-11-25T12:55:06.737471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids = os.listdir(train_image_path)[13:16]\nfor image_id in image_ids:\n    path = os.path.join(train_image_path,image_id)\n    img = cv2.imread(path)\n    img =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    masks = get_masks_by_img_id(train_df, image_id)\n    masks = (masks[:,:,0], masks[:,:,1],masks[:,:,2],masks[:,:,3])\n    show_img_with_masks(img,masks)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:06.739523Z","iopub.execute_input":"2023-11-25T12:55:06.739815Z","iopub.status.idle":"2023-11-25T12:55:33.759715Z","shell.execute_reply.started":"2023-11-25T12:55:06.739789Z","shell.execute_reply":"2023-11-25T12:55:33.758600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Custom Data Generator  <a class=\"anchor\" id=\"data_gen\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{"execution":{"iopub.status.busy":"2023-06-03T00:01:18.601333Z","iopub.execute_input":"2023-06-03T00:01:18.602029Z","iopub.status.idle":"2023-06-03T00:01:18.606731Z","shell.execute_reply.started":"2023-06-03T00:01:18.602004Z","shell.execute_reply":"2023-06-03T00:01:18.605097Z"}}},{"cell_type":"code","source":"def rle_to_mask(rle_string, height, width):\n    '''\n    convert RLE(run length encoding) string to numpy array\n\n    Parameters: \n    rle_string (str): string of rle encoded mask\n    height (int): height of the mask\n    width (int): width of the mask \n\n    Returns: \n    numpy.array: numpy array of the mask\n    '''\n    \n    rows, cols = height, width\n    \n    if rle_string == -1:\n        return np.zeros((height,width))\n    else:\n        rle_numbers = [int(num_string) for num_string in rle_string.split(' ')]\n        rle_pairs = np.array(rle_numbers).reshape(-1,2)\n        img = np.zeros(rows*cols, dtype=np.uint8)\n        for index, length in rle_pairs:\n            index -= 1\n            img[index:index+length] = 255\n        img = img.reshape(cols,rows)\n        img = img.T\n        img = img/255.0\n        return img\n    \ndef get_masks_by_img_id(dataframe, image_id):\n    masks = np.zeros((img_height,img_width,4))\n    rle_masks = list(dataframe[dataframe['Image_Id'] == image_id]['Label_EncodedPixels'])[0]\n    fish_mask = rle_to_mask(rle_masks[0][1], img_height, img_width)\n    flower_mask = rle_to_mask(rle_masks[1][1], img_height, img_width)\n    gravel_mask = rle_to_mask(rle_masks[2][1], img_height, img_width)\n    sugar_mask = rle_to_mask(rle_masks[3][1], img_height, img_width)\n    mask_list = [fish_mask,flower_mask,gravel_mask,sugar_mask]\n    for ix, mask in enumerate(mask_list):\n        masks[:,:,ix] = mask\n    return masks\n\ndef draw_label_on_mask(mask, label, obj=plt):\n    '''\n    Function to add labels to the image.\n    '''\n    if np.sum(mask) > 0:\n        y,x = 0,0\n        y,x = np.argwhere(mask==1)[0]\n        y,x = y+10,x+5      \n        obj.text(x,y,label,color='white',)\n    return None\n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:33.762229Z","iopub.execute_input":"2023-11-25T12:55:33.762530Z","iopub.status.idle":"2023-11-25T12:55:33.774734Z","shell.execute_reply.started":"2023-11-25T12:55:33.762504Z","shell.execute_reply":"2023-11-25T12:55:33.773797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport math\n\nclass DataGenerator(tf.keras.utils.Sequence):\n    \n    def __init__(self,\n                 dataframe=None,\n                 root_dir=\".\",\n                 img_width=2100,\n                 img_height=1400,\n                 resize = False,\n                 resize_width=384,\n                 resize_height=256,\n                 mode='fit',\n                 augmentations=None,\n                 num_channels = 3,\n                 num_classes = 4,\n                 batch_size=32,\n                 shuffle=True, \n                 random_state=42): \n        \n        self.dataframe = dataframe\n        self.filenames = list(dataframe['Image_Id']) if not dataframe is None else os.listdir(root_dir)\n        self.root_dir = root_dir\n        self.img_width = img_width\n        self.img_height = img_height\n        self.resize = resize\n        self.resize_width = resize_width\n        self.resize_height = resize_height\n        self.mode = mode\n        self.augmentations = augmentations\n        self.num_channels = num_channels\n        self.num_classes = num_classes\n        self.total_samples = len(self.filenames)\n        self.indexes = np.arange(len(self.filenames))\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.random_state = random_state\n        \n        self.on_epoch_end()\n        \n    @property\n    def image_shape(self):\n        if not self.resize:\n            img_shape = (self.img_height, self.img_width, self.num_channels)\n        else:\n            img_shape = (self.resize_height, self.resize_width, self.num_channels)\n        return img_shape\n    \n    \n    def __len__(self):\n        num_batches_total = math.ceil(self.total_samples / self.batch_size)\n        return num_batches_total\n    \n    def __getitem__(self,index):\n        low = index * self.batch_size\n        high = min(low + self.batch_size, self.total_samples)\n        batch_files = self.filenames[low:high]\n        \n        if self.mode == 'fit':\n            batch_X = self.__generate_X(batch_files)\n            batch_y = self.__generate_y(batch_files)\n            if self.augmentations is not None:\n                batch_X, batch_y = self.augment_batch(batch_X, batch_y)\n            batch_X, batch_y = batch_X.astype(np.float32), batch_y.astype(np.float32)\n            return batch_X, batch_y\n        elif self.mode == 'predict':\n            batch_X = self.__generate_X(batch_files)\n            batch_X = batch_X.astype(np.float32)\n            return batch_X\n        else:\n            raise AttributeError('The mode parameter should be set to \"fit\" or \"predict\".')\n    \n    def __generate_X(self,batch_files):\n        if self.resize:\n            img_size = (self.resize_height, self.resize_width)\n        else:\n            img_size = (self.img_height, self.img_width)\n            \n        batch_images = np.zeros((len(batch_files),*img_size, self.num_channels), dtype=np.uint8)\n\n        for ix,filename in enumerate(batch_files):\n            img_path = os.path.join(self.root_dir, filename)\n            if self.num_channels==3:\n                img = cv2.imread(img_path)\n                img =  cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            else:\n                img = cv2.imread(img_path,0)\n                #img = np.expand_dims(img, axis=-1)\n            if self.resize:\n                img = cv2.resize(img,tuple(reversed(img_size)))\n                if self.num_channels==1:\n                    img = np.expand_dims(img, axis=-1)\n                        \n            img = img.astype(np.uint8)\n            #img = img/255\n            batch_images[ix] = img\n                               \n        return  batch_images\n    \n    def __generate_y(self,batch_files):\n        if self.resize:\n            mask_size = (self.resize_height, self.resize_width)\n        else:\n            mask_size = (self.img_height, self.img_width)\n        \n        batch_masks = np.zeros((len(batch_files),*mask_size, self.num_classes), dtype = np.uint8)\n        \n        for ix,filename in  enumerate(batch_files):\n            masks = self.get_masks_by_img_id(self.dataframe, filename)\n            batch_masks[ix] = masks\n        \n        batch_masks = (batch_masks > 0).astype(int)\n        batch_masks = batch_masks.astype(np.uint8)\n        return batch_masks\n    \n                                 \n    def on_epoch_end(self):\n        self.indexes = np.arange(self.total_samples)\n        if self.shuffle:\n            np.random.seed(self.random_state)\n            np.random.shuffle(self.indexes)\n            \n\n    def get_masks_by_img_id(self,dataframe,image_id):\n        rle_masks = list(dataframe[dataframe['Image_Id'] == image_id]['Label_EncodedPixels'])[0]\n        fish_mask = rle_to_mask(rle_masks[0][1], self.img_height, self.img_width)\n        flower_mask = rle_to_mask(rle_masks[1][1], self.img_height, self.img_width)\n        gravel_mask = rle_to_mask(rle_masks[2][1], self.img_height, self.img_width)\n        sugar_mask = rle_to_mask(rle_masks[3][1], self.img_height, self.img_width)\n        mask_list = [fish_mask,flower_mask,gravel_mask,sugar_mask]\n        if self.resize:\n            resized_mask_list = []\n            for mask in mask_list:\n                mask = cv2.resize(mask, (self.resize_width,self.resize_height))\n                resized_mask_list.append(mask)\n            masks = np.zeros((self.resize_height,self.resize_width,self.num_classes))\n            \n            for ix, mask in enumerate(resized_mask_list):\n                masks[:,:,ix] = mask\n        else:\n            masks = np.zeros((self.img_height,self.img_width,self.num_classes))   \n            for ix, mask in enumerate(mask_list):\n                masks[:,:,ix] = mask\n                \n        return masks\n    \n\n    def augment_batch(self,batch_images, batch_masks):\n        batch_X = np.zeros(batch_images.shape, dtype=np.float32)\n        batch_y = np.zeros(batch_masks.shape, dtype=np.float32)\n        for ix, (img,masks) in enumerate(zip(batch_images,batch_masks)):\n            augmented = self.augmentations(image=img, mask=masks)\n            img = augmented['image']\n            masks = augmented['mask']\n            batch_X[ix] = img\n            batch_y[ix] = masks\n        return batch_X, batch_y\n        \n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:33.776162Z","iopub.execute_input":"2023-11-25T12:55:33.776654Z","iopub.status.idle":"2023-11-25T12:55:33.815505Z","shell.execute_reply.started":"2023-11-25T12:55:33.776620Z","shell.execute_reply":"2023-11-25T12:55:33.814728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"augmentations = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.5),\n    A.ShiftScaleRotate(p=0.5),\n    A.GridDistortion(p=0.5),\n    A.OpticalDistortion(p=0.5),\n    A.RandomGamma(),\n    A.RandomBrightnessContrast(),\n    A.CLAHE(),\n    A.Equalize(),\n    A.ToFloat()\n])","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:33.816528Z","iopub.execute_input":"2023-11-25T12:55:33.816812Z","iopub.status.idle":"2023-11-25T12:55:33.829543Z","shell.execute_reply.started":"2023-11-25T12:55:33.816789Z","shell.execute_reply":"2023-11-25T12:55:33.828701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_generator = DataGenerator(train_df,\n                               train_image_path,\n                               resize=True,\n                               resize_width=576,\n                               resize_height=384,\n                               num_channels=3,\n                               augmentations=augmentations\n                                )\n\nprint(data_generator.total_samples)\nprint(len(data_generator.indexes))\nprint(data_generator.__len__())\nprint(data_generator.image_shape)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:33.830671Z","iopub.execute_input":"2023-11-25T12:55:33.830999Z","iopub.status.idle":"2023-11-25T12:55:33.846133Z","shell.execute_reply.started":"2023-11-25T12:55:33.830974Z","shell.execute_reply":"2023-11-25T12:55:33.845209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_X, batch_y = data_generator.__getitem__(1)\nprint(batch_X.shape)\nprint(batch_y.shape)\nprint(batch_X.dtype)\nprint(batch_y.dtype)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:33.847236Z","iopub.execute_input":"2023-11-25T12:55:33.847533Z","iopub.status.idle":"2023-11-25T12:55:37.931648Z","shell.execute_reply.started":"2023-11-25T12:55:33.847510Z","shell.execute_reply":"2023-11-25T12:55:37.930684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(batch_X[13])","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:37.932981Z","iopub.execute_input":"2023-11-25T12:55:37.933346Z","iopub.status.idle":"2023-11-25T12:55:38.274825Z","shell.execute_reply.started":"2023-11-25T12:55:37.933310Z","shell.execute_reply":"2023-11-25T12:55:38.273890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(batch_y[13][:,:,1])","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:38.276401Z","iopub.execute_input":"2023-11-25T12:55:38.276723Z","iopub.status.idle":"2023-11-25T12:55:38.514120Z","shell.execute_reply.started":"2023-11-25T12:55:38.276694Z","shell.execute_reply":"2023-11-25T12:55:38.513214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for img,masks in zip(batch_X[3:6],batch_y[3:6]):\n    masks = (masks[:,:,0], masks[:,:,1],masks[:,:,2],masks[:,:,3])\n    show_img_with_masks(img,masks)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:38.515234Z","iopub.execute_input":"2023-11-25T12:55:38.515489Z","iopub.status.idle":"2023-11-25T12:55:43.012636Z","shell.execute_reply.started":"2023-11-25T12:55:38.515467Z","shell.execute_reply":"2023-11-25T12:55:43.010733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Test Split <a class=\"anchor\" id=\"data_split\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"code","source":"df_train, df_val = train_test_split(train_df,test_size=0.1,random_state=42, stratify=train_df['classes'])\nprint(df_train.shape)\nprint(df_val.shape)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:43.016346Z","iopub.execute_input":"2023-11-25T12:55:43.016720Z","iopub.status.idle":"2023-11-25T12:55:43.041741Z","shell.execute_reply.started":"2023-11-25T12:55:43.016693Z","shell.execute_reply":"2023-11-25T12:55:43.040637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[labels].sum().plot(kind='bar')\nplt.xlabel('Columns')\nplt.ylabel('Frequency')","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:43.043030Z","iopub.execute_input":"2023-11-25T12:55:43.043415Z","iopub.status.idle":"2023-11-25T12:55:43.307868Z","shell.execute_reply.started":"2023-11-25T12:55:43.043304Z","shell.execute_reply":"2023-11-25T12:55:43.306952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_val[labels].sum().plot(kind='bar')\nplt.xlabel('Columns')\nplt.ylabel('Frequency')","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:55:43.309185Z","iopub.execute_input":"2023-11-25T12:55:43.309571Z","iopub.status.idle":"2023-11-25T12:55:43.510049Z","shell.execute_reply.started":"2023-11-25T12:55:43.309537Z","shell.execute_reply":"2023-11-25T12:55:43.509178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"R_WIDTH = 576\nR_HEIGHT = 384\nNUM_CHANNELS = 3\nBATCH_SIZE = 4\nTEST_BATCH_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:14:59.607167Z","iopub.execute_input":"2023-11-25T13:14:59.607932Z","iopub.status.idle":"2023-11-25T13:15:00.672531Z","shell.execute_reply.started":"2023-11-25T13:14:59.607878Z","shell.execute_reply":"2023-11-25T13:15:00.671345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = DataGenerator(dataframe=df_train,\n                                root_dir=train_image_path,\n                                mode=\"fit\",\n                                resize=True,\n                                shuffle=True,\n                                resize_width=R_WIDTH,\n                                resize_height=R_HEIGHT,\n                                num_channels=NUM_CHANNELS,\n                                batch_size=BATCH_SIZE,\n                                augmentations=augmentations)\n\nprint(train_generator.total_samples)\nprint(len(train_generator.indexes))\nprint(train_generator.__len__())\n\nbatch_X, batch_y = train_generator.__getitem__(4)\nprint(batch_X.shape)\nprint(batch_y.shape)\nprint(batch_X.dtype)\nprint(batch_y.dtype)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:15:00.674496Z","iopub.execute_input":"2023-11-25T13:15:00.674838Z","iopub.status.idle":"2023-11-25T13:15:02.558215Z","shell.execute_reply.started":"2023-11-25T13:15:00.674804Z","shell.execute_reply":"2023-11-25T13:15:02.556972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_generator =  DataGenerator(dataframe=df_val,\n                               root_dir=train_image_path,\n                               mode=\"fit\",\n                               resize=True,\n                               shuffle=True,\n                               resize_width=R_WIDTH,\n                               resize_height=R_HEIGHT,\n                               num_channels=NUM_CHANNELS,\n                               batch_size=BATCH_SIZE,\n                               augmentations=augmentations)\n\nprint(val_generator.total_samples)\nprint(len(val_generator.indexes))\nprint(val_generator.__len__())\n\nbatch_X, batch_y = val_generator.__getitem__(1)\nprint(batch_X.shape)\nprint(batch_y.shape)\nprint(batch_X.dtype)\nprint(batch_y.dtype)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:15:02.560118Z","iopub.execute_input":"2023-11-25T13:15:02.560518Z","iopub.status.idle":"2023-11-25T13:15:04.403334Z","shell.execute_reply.started":"2023-11-25T13:15:02.560480Z","shell.execute_reply":"2023-11-25T13:15:04.401742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_generator =  DataGenerator(dataframe=test_df,\n                                root_dir=test_image_path,\n                                mode=\"predict\",\n                                resize=True,\n                                shuffle=False,\n                                resize_width=R_WIDTH,\n                                resize_height=R_HEIGHT,\n                                num_channels=NUM_CHANNELS,\n                                batch_size=TEST_BATCH_SIZE)\n\n\nprint(test_generator.total_samples)\nprint(len(test_generator.indexes))\nprint(test_generator.__len__())\n\nbatch_X = test_generator.__getitem__(1)\nprint(batch_X.shape)\nprint(batch_X.dtype)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:15:04.405264Z","iopub.execute_input":"2023-11-25T13:15:04.405648Z","iopub.status.idle":"2023-11-25T13:15:06.825085Z","shell.execute_reply.started":"2023-11-25T13:15:04.405605Z","shell.execute_reply":"2023-11-25T13:15:06.824033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_generator =  DataGenerator(dataframe=df_val,\n                               root_dir=train_image_path,\n                               mode=\"fit\",\n                               resize=True,\n                               shuffle=False,\n                               resize_width=R_WIDTH,\n                               resize_height=R_HEIGHT,\n                               num_channels=NUM_CHANNELS,\n                               batch_size=TEST_BATCH_SIZE,\n                               augmentations=augmentations)\n\nprint(eval_generator.total_samples)\nprint(len(eval_generator.indexes))\nprint(eval_generator.__len__())\n\nbatch_X, batch_y = eval_generator.__getitem__(1)\nprint(batch_X.shape)\nprint(batch_y.shape)\nprint(batch_X.dtype)\nprint(batch_y.dtype)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:15:06.827046Z","iopub.execute_input":"2023-11-25T13:15:06.827320Z","iopub.status.idle":"2023-11-25T13:15:12.765710Z","shell.execute_reply.started":"2023-11-25T13:15:06.827296Z","shell.execute_reply":"2023-11-25T13:15:12.764569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Definitions <a class=\"anchor\" id=\"model_def\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nimport io\n\ndef viz_model(model):\n    tf.keras.backend.clear_session()\n    model_plot = tf.keras.utils.plot_model(model, show_shapes=True, rankdir='TB')\n    model_plot = io.BytesIO(model_plot.data)\n    model_plot = Image.open(model_plot)\n    model_plot = np.array(model_plot)\n    model_plot = np.transpose(model_plot, (1, 0, 2))\n    model_plot = Image.fromarray(model_plot)\n    model_plot = model_plot.transpose(Image.FLIP_TOP_BOTTOM)\n    model_plot = np.array(model_plot)\n    fig = plt.figure(figsize=(20,10))\n    ax = fig.add_subplot(111)\n    ax.imshow(model_plot)\n    ax.axis('off')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:56:05.757351Z","iopub.execute_input":"2023-11-25T12:56:05.758167Z","iopub.status.idle":"2023-11-25T12:56:05.765478Z","shell.execute_reply.started":"2023-11-25T12:56:05.758133Z","shell.execute_reply":"2023-11-25T12:56:05.764473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Vanilla Unet OOP implementation","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\nclass UNet:\n    def __init__(self,\n                 input_shape=(256,384,1),\n                 num_classes=4,\n                 activation='relu',\n                 num_filters=(32,64,128,256,512)):\n        \n        self.input_shape = input_shape\n        self.num_classes = num_classes\n        self.activation = activation\n        self.num_filters =  num_filters\n        self.model = self.build()\n\n    def conv_block(self, inputs, filters, kernel_size=3, activation='relu', padding='same'):\n        conv = tf.keras.layers.Conv2D(filters, kernel_size, padding=padding)(inputs)\n        conv = tf.keras.layers.BatchNormalization()(conv)\n        conv = tf.keras.layers.Activation(activation)(conv)\n        conv = tf.keras.layers.Conv2D(filters, kernel_size, padding=padding)(conv)\n        conv = tf.keras.layers.BatchNormalization()(conv)\n        conv = tf.keras.layers.Activation(activation)(conv)\n        return conv\n\n    def upsample_block(self, inputs, skip_connection, filters, kernel_size=2, activation='relu', padding='same'):\n        upsample = tf.keras.layers.Conv2DTranspose(filters, kernel_size, strides=(2, 2), padding=padding)(inputs)\n        merge = tf.keras.layers.concatenate([skip_connection, upsample], axis=3)\n        conv = self.conv_block(merge, filters, kernel_size, activation, padding)\n        return conv\n\n    def build(self):\n        # Input layer\n        inputs = tf.keras.layers.Input(self.input_shape)\n\n        # Contracting path\n        conv1 = self.conv_block(inputs, self.num_filters[0],activation=self.activation)\n        pool1 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(conv1)\n\n        conv2 = self.conv_block(pool1, self.num_filters[1],activation=self.activation)\n        pool2 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(conv2)\n\n        conv3 = self.conv_block(pool2, self.num_filters[2],activation=self.activation)\n        pool3 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(conv3)\n\n        conv4 = self.conv_block(pool3, self.num_filters[3],activation=self.activation)\n        pool4 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2))(conv4)\n\n        conv5 = self.conv_block(pool4, self.num_filters[4],activation=self.activation)\n        \n\n        # Expansive path\n        up6 = self.upsample_block(conv5, conv4, self.num_filters[3], activation=self.activation)\n        up7 = self.upsample_block(up6, conv3, self.num_filters[2], activation=self.activation)\n        up8 = self.upsample_block(up7, conv2, self.num_filters[1], activation=self.activation)\n        up9 = self.upsample_block(up8, conv1, self.num_filters[0], activation=self.activation)\n\n        # Output layer\n        outputs = tf.keras.layers.Conv2D(self.num_classes, 1, activation='sigmoid')(up9)\n\n        # Create model\n        model = tf.keras.Model(inputs=inputs, outputs=outputs)\n        return model\n\n    \n    def get_model(self):\n        return self.model","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:56:06.799430Z","iopub.execute_input":"2023-11-25T12:56:06.799776Z","iopub.status.idle":"2023-11-25T12:56:06.817148Z","shell.execute_reply.started":"2023-11-25T12:56:06.799750Z","shell.execute_reply":"2023-11-25T12:56:06.816169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tf.keras.backend.clear_session()\n#UNet().get_model().summary()\nmodel = UNet().get_model()\ntotal_param = format(model.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\nviz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:56:07.571623Z","iopub.execute_input":"2023-11-25T12:56:07.571997Z","iopub.status.idle":"2023-11-25T12:56:10.972168Z","shell.execute_reply.started":"2023-11-25T12:56:07.571967Z","shell.execute_reply":"2023-11-25T12:56:10.971192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.Xception Style Unet from Keras Documentation OOP Implementation \nhttps://keras.io/examples/vision/oxford_pets_image_segmentation/","metadata":{}},{"cell_type":"code","source":"class UnetXception:\n    def __init__(self,\n                 input_shape=(256,384, 1),\n                 num_classes=4,\n                 activation='relu',\n                 num_filters=(32,64,128,256)):\n        \n        self.input_shape = input_shape\n        self.num_classes = num_classes\n        self.activation = activation\n        self.num_filters =  num_filters\n        self.model = self.build()\n\n    def build(self):\n        inputs = tf.keras.layers.Input(self.input_shape)\n        x = self.downsampling(inputs)\n        x = self.upsampling(x)\n        outputs = tf.keras.layers.Conv2D(self.num_classes, 1, activation=\"sigmoid\", padding=\"same\")(x)\n        model = tf.keras.Model(inputs, outputs)\n        return model\n\n    def downsampling(self, x):\n        # Entry block\n        x = tf.keras.layers.Conv2D(32, 3, strides=2, padding=\"same\")(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = tf.keras.layers.Activation(\"relu\")(x)\n        previous_block_activation = x\n        \n        # Blocks 1, 2, 3 are identical apart from the feature depth.\n        for filters in self.num_filters[1:]:\n            x = tf.keras.layers.Activation(\"relu\")(x)\n            x = tf.keras.layers.SeparableConv2D(filters, 3, padding=\"same\")(x)\n            x = tf.keras.layers.BatchNormalization()(x)\n\n            x = tf.keras.layers.Activation(\"relu\")(x)\n            x = tf.keras.layers.SeparableConv2D(filters, 3, padding=\"same\")(x)\n            x = tf.keras.layers.BatchNormalization()(x)\n\n            x = tf.keras.layers.MaxPooling2D(3, strides=2, padding=\"same\")(x)\n\n            # Project residual\n            residual =  tf.keras.layers.Conv2D(filters, 1, strides=2, padding=\"same\")(\n                previous_block_activation\n            )\n            x =  tf.keras.layers.add([x, residual])  # Add back residual\n            previous_block_activation = x  # Set aside next residual\n\n        return x\n\n    \n    def upsampling(self, x):\n        num_filters = list(reversed(self.num_filters))\n        previous_block_activation = x\n        for filters in num_filters:\n            x = tf.keras.layers.Activation(\"relu\")(x)\n            x = tf.keras.layers.Conv2DTranspose(filters, 3, padding=\"same\")(x)\n            x = tf.keras.layers.BatchNormalization()(x)\n\n            x = tf.keras.layers.Activation(\"relu\")(x)\n            x = tf.keras.layers.Conv2DTranspose(filters, 3, padding=\"same\")(x)\n            x = tf.keras.layers.BatchNormalization()(x)\n\n            x = tf.keras.layers.UpSampling2D(2)(x)\n\n            # Project residual\n            residual =  tf.keras.layers.UpSampling2D(2)(previous_block_activation)\n            residual =  tf.keras.layers.Conv2D(filters, 1, padding=\"same\")(residual)\n            x =  tf.keras.layers.add([x, residual])  # Add back residual\n            previous_block_activation = x  # Set aside next residual\n\n        return x\n\n\n    def get_model(self):\n        return self.model\n\n    ","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:56:16.342484Z","iopub.execute_input":"2023-11-25T12:56:16.343335Z","iopub.status.idle":"2023-11-25T12:56:16.360255Z","shell.execute_reply.started":"2023-11-25T12:56:16.343301Z","shell.execute_reply":"2023-11-25T12:56:16.359324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tf.keras.backend.clear_session()\n#UnetXception.get_model().summary()\nmodel = UnetXception().get_model()\ntotal_param = format(model.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\nviz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:56:19.973059Z","iopub.execute_input":"2023-11-25T12:56:19.973889Z","iopub.status.idle":"2023-11-25T12:56:22.802575Z","shell.execute_reply.started":"2023-11-25T12:56:19.973849Z","shell.execute_reply":"2023-11-25T12:56:22.801671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. Unet with ResNet50 backbone(encoder)","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\nclass UNetResNet50:\n    def __init__(self,\n                 input_shape=(256,384, 3),\n                 num_classes=4,\n                 activation='relu'):\n        self.input_shape = input_shape\n        self.num_classes = num_classes\n        self.activation = activation\n        self.model = self.build()\n\n    def build_encoder(self, input_tensor):\n        base_model = tf.keras.applications.ResNet50(weights='imagenet', include_top=False, input_tensor=input_tensor)\n        return base_model\n\n    def build_decoder(self, encoder_output):\n        decoder = tf.keras.layers.Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same')(encoder_output.output)\n        decoder = tf.keras.layers.Concatenate()([decoder, encoder_output.layers[142].output])  # Skip connection\n        decoder = tf.keras.layers.Conv2D(256, (3, 3), activation=self.activation, padding='same')(decoder)\n        decoder = tf.keras.layers.Conv2D(256, (3, 3), activation=self.activation, padding='same')(decoder)\n\n        decoder = tf.keras.layers.Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(decoder)\n        decoder = tf.keras.layers.Concatenate()([decoder, encoder_output.layers[80].output])  # Skip connection\n        decoder = tf.keras.layers.Conv2D(128, (3, 3), activation=self.activation, padding='same')(decoder)\n        decoder = tf.keras.layers.Conv2D(128, (3, 3), activation=self.activation, padding='same')(decoder)\n\n        decoder = tf.keras.layers.Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(decoder)\n        decoder = tf.keras.layers.Concatenate()([decoder, encoder_output.layers[38].output])  # Skip connection\n        decoder = tf.keras.layers.Conv2D(64, (3, 3), activation=self.activation, padding='same')(decoder)\n        decoder = tf.keras.layers.Conv2D(64, (3, 3), activation=self.activation, padding='same')(decoder)\n\n        decoder = tf.keras.layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(decoder)\n        decoder = tf.keras.layers.Concatenate()([decoder, encoder_output.layers[4].output])  # Skip connection\n        decoder = tf.keras.layers.Conv2D(32, (3, 3), activation=self.activation, padding='same')(decoder)\n        decoder = tf.keras.layers.Conv2D(32, (3, 3), activation=self.activation, padding='same')(decoder)\n\n        return decoder\n\n    def build(self):\n        input_tensor = tf.keras.layers.Input(shape=self.input_shape)\n        encoder_output = self.build_encoder(input_tensor)\n        decoder_output = self.build_decoder(encoder_output)\n        output = tf.keras.layers.Conv2D(self.num_classes, (1, 1), activation='sigmoid')(decoder_output)\n        model = tf.keras.models.Model(inputs=input_tensor, outputs=output)\n        return model\n\n    \n    def get_model(self):\n        return self.model","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:56:39.436583Z","iopub.execute_input":"2023-11-25T12:56:39.437296Z","iopub.status.idle":"2023-11-25T12:56:39.454573Z","shell.execute_reply.started":"2023-11-25T12:56:39.437257Z","shell.execute_reply":"2023-11-25T12:56:39.453495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tf.keras.backend.clear_session()\n#UnetXception.get_model().summary()\nmodel = UNetResNet50().get_model()\ntotal_param = format(model.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\nviz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:56:40.042675Z","iopub.execute_input":"2023-11-25T12:56:40.043424Z","iopub.status.idle":"2023-11-25T12:56:50.780052Z","shell.execute_reply.started":"2023-11-25T12:56:40.043389Z","shell.execute_reply":"2023-11-25T12:56:50.779138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. Unet with EfficientNetB2 backbone(encoder)","metadata":{}},{"cell_type":"code","source":"class UNetEfficientNetB2:\n    def __init__(self,\n                 input_shape=(256,384, 3),\n                 num_classes=4,\n                 activation='relu'):\n        self.input_shape = input_shape\n        self.num_classes = num_classes\n        self.activation = activation\n        self.model = self.build_model()\n\n    def build_encoder(self, input_tensor):\n        base_model =  tf.keras.applications.EfficientNetB2(weights='imagenet', include_top=False, input_shape=input_shape)\n        return base_model\n\n    def build_decoder(self, encoder_output):\n        decoder = tf.keras.layers.Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same')(encoder_output.output)\n        decoder = tf.keras.layers.Concatenate()([decoder, encoder_output.layers[254].output])  # Skip connection\n        decoder = tf.keras.layers.Conv2D(256, (3, 3), activation=self.activation, padding='same')(decoder)\n        decoder = tf.keras.layers.Conv2D(256, (3, 3), activation=self.activation, padding='same')(decoder)\n\n        decoder = tf.keras.layers.Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(decoder)\n        decoder = tf.keras.layers.Concatenate()([decoder, encoder_output.layers[122].output])  # Skip connection\n        decoder = tf.keras.layers.Conv2D(128, (3, 3), activation=self.activation, padding='same')(decoder)\n        decoder = tf.keras.layers.Conv2D(128, (3, 3), activation=self.activation, padding='same')(decoder)\n\n        decoder = tf.keras.layers.Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(decoder)\n        decoder = tf.keras.layers.Concatenate()([decoder, encoder_output.layers[50].output])  # Skip connection\n        decoder = tf.keras.layers.Conv2D(64, (3, 3), activation=self.activation, padding='same')(decoder)\n        decoder = tf.keras.layers.Conv2D(64, (3, 3), activation=self.activation, padding='same')(decoder)\n\n        decoder = tf.keras.layers.Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(decoder)\n        decoder = tf.keras.layers.Concatenate()([decoder, encoder_output.layers[7].output])  # Skip connection\n        decoder = tf.keras.layers.Conv2D(32, (3, 3), activation=self.activation, padding='same')(decoder)\n        decoder = tf.keras.layers.Conv2D(32, (3, 3), activation=self.activation, padding='same')(decoder)\n\n        return decoder\n\n    def build_model(self):\n        input_tensor =  tf.keras.layers.Input(shape=self.input_shape)\n        encoder_output = self.build_encoder(input_tensor)\n        decoder_output = self.build_decoder(encoder_output)\n        output = tf.keras.layers.Conv2D(self.num_classes, (1, 1), activation='sigmoid')(decoder_output)\n        model = Model(inputs=input_tensor, outputs=output)\n        return model\n    \n    def get_model(self):\n        return self.model\n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:56:50.781777Z","iopub.execute_input":"2023-11-25T12:56:50.782075Z","iopub.status.idle":"2023-11-25T12:56:50.799895Z","shell.execute_reply.started":"2023-11-25T12:56:50.782050Z","shell.execute_reply":"2023-11-25T12:56:50.798961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tf.keras.backend.clear_session()\n#UnetXception.get_model().summary()\n#model = UNetEfficientNetB2().get_model()\n#total_param = format(model.count_params(),\",\")\n#print(f\"Total no of parameters = {total_param}\")\n#viz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:56:50.801219Z","iopub.execute_input":"2023-11-25T12:56:50.801855Z","iopub.status.idle":"2023-11-25T12:56:50.814108Z","shell.execute_reply.started":"2023-11-25T12:56:50.801820Z","shell.execute_reply":"2023-11-25T12:56:50.813106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5. Unet with different backbones using Segmentation-Models Library ","metadata":{}},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = sm.Unet('efficientnetb3',\n                input_shape=(256,384, 1),\n                classes=4,\n                activation='sigmoid',\n                encoder_weights=None)\n\n\ntotal_param = format(model.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\nviz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:56:50.816125Z","iopub.execute_input":"2023-11-25T12:56:50.816422Z","iopub.status.idle":"2023-11-25T12:57:00.799890Z","shell.execute_reply.started":"2023-11-25T12:56:50.816397Z","shell.execute_reply":"2023-11-25T12:57:00.798934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = sm.Unet('resnet50',\n                input_shape=(256,384, 1),\n                classes=4,\n                activation='sigmoid',\n                encoder_weights=None)\n\n\ntotal_param = format(model.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\nviz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:57:00.801192Z","iopub.execute_input":"2023-11-25T12:57:00.801579Z","iopub.status.idle":"2023-11-25T12:57:10.846225Z","shell.execute_reply.started":"2023-11-25T12:57:00.801544Z","shell.execute_reply":"2023-11-25T12:57:10.845320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = sm.FPN('efficientnetb2',\n                input_shape=(256,384, 1),\n                classes=4,\n                activation='sigmoid',\n                encoder_weights=None)\n\n\ntotal_param = format(model.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\nviz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:57:10.848261Z","iopub.execute_input":"2023-11-25T12:57:10.848963Z","iopub.status.idle":"2023-11-25T12:57:28.352992Z","shell.execute_reply.started":"2023-11-25T12:57:10.848924Z","shell.execute_reply":"2023-11-25T12:57:28.352066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6. DeepLabV3+ with self attention","metadata":{}},{"cell_type":"markdown","source":"1. https://github.com/taki0112/Self-Attention-GAN-Tensorflow/\n2. https://link.springer.com/article/10.1007/s10489-020-01917-8\n3. https://arxiv.org/abs/1905.08008","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:57:32.481027Z","iopub.execute_input":"2023-11-25T12:57:32.481377Z","iopub.status.idle":"2023-11-25T12:57:32.485469Z","shell.execute_reply.started":"2023-11-25T12:57:32.481350Z","shell.execute_reply":"2023-11-25T12:57:32.484570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base = tf.keras.applications.EfficientNetB0(include_top=False,\n                                            weights=\"imagenet\",\n                                            input_shape=(256,384, 3)\n                                            )","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:57:32.930069Z","iopub.execute_input":"2023-11-25T12:57:32.930439Z","iopub.status.idle":"2023-11-25T12:57:35.130767Z","shell.execute_reply.started":"2023-11-25T12:57:32.930409Z","shell.execute_reply":"2023-11-25T12:57:35.129719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n#base.summary()\ntotal_param = format(base.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\nviz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:57:55.280235Z","iopub.execute_input":"2023-11-25T12:57:55.281075Z","iopub.status.idle":"2023-11-25T12:58:10.571284Z","shell.execute_reply.started":"2023-11-25T12:57:55.281041Z","shell.execute_reply":"2023-11-25T12:58:10.570444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n#tf.keras.utils.plot_model(base, show_shapes=True, rankdir='TB')","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:57:37.738496Z","iopub.execute_input":"2023-11-25T12:57:37.738885Z","iopub.status.idle":"2023-11-25T12:57:37.746596Z","shell.execute_reply.started":"2023-11-25T12:57:37.738852Z","shell.execute_reply":"2023-11-25T12:57:37.745671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base.get_layer('block7a_dwconv').output[0]\nbase.get_layer('block6d_project_conv').output[0]\nbase.get_layer('block7a_bn').output[0]\nbase.get_layer('block2a_dwconv').output[0]","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:58:25.717455Z","iopub.execute_input":"2023-11-25T12:58:25.717802Z","iopub.status.idle":"2023-11-25T12:58:25.745819Z","shell.execute_reply.started":"2023-11-25T12:58:25.717776Z","shell.execute_reply":"2023-11-25T12:58:25.744968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def hw_flatten(x) :\n    return tf.keras.layers.Reshape(target_shape=(-1,x.shape[-1]))(x)\n\n\ndef self_attention_cnn(x):\n    q = tf.keras.layers.Conv2D(filters=x.shape[-1]//8, kernel_size=(1,1))(x) # [bs, h, w, c']\n    k = tf.keras.layers.Conv2D(filters=x.shape[-1]//8, kernel_size=(1,1))(x) # [bs, h, w, c']\n    v = tf.keras.layers.Conv2D(filters=x.shape[-1], kernel_size=(1,1))(x) # [bs, h, w, c]\n    qk = tf.keras.layers.Dot(axes=2)([hw_flatten(q), hw_flatten(k)]) # [bs, n, n] ;n = h * w\n    softmax = tf.keras.layers.Activation(\"softmax\")(qk) # [bs, n, n]\n    attention = tf.keras.layers.Dot(axes=1)([softmax,hw_flatten(v)]) # [bs, N, c]\n    attention = tf.keras.layers.Reshape(target_shape=tuple(x.shape[1:]))(attention) # [bs, h,w, c]\n    y = tf.keras.layers.Add()([x,attention]) # [bs, h,w, c]\n    return y\n\ndef refined_self_attention_cnn(x):\n    q = tf.keras.layers.Conv2D(filters=x.shape[-1]//8, kernel_size=(1,1))(x) # [bs, h, w, c']\n    k = tf.keras.layers.Conv2D(filters=x.shape[-1]//8, kernel_size=(1,1))(x) # [bs, h, w, c']\n    v = tf.keras.layers.Conv2D(filters=x.shape[-1], kernel_size=(1,1))(x) # [bs, h, w, c]\n    kv = tf.keras.layers.Dot(axes=1)([hw_flatten(k), hw_flatten(v)])\n    norm = kv/hw_flatten(x).shape[1]\n    norm = tf.keras.layers.Reshape((norm.shape[-1],norm.shape[1]))(norm)\n    attention = tf.keras.layers.Dot(axes=2)([hw_flatten(q),norm])\n    attention = tf.keras.layers.Reshape(target_shape=tuple(x.shape[1:]))(attention)\n    y = tf.keras.layers.Add()([x,attention])\n    return x\n\n\ndef ASPP(inputs):\n    shape = inputs.shape\n\n    y_pool = tf.keras.layers.AveragePooling2D(pool_size=(shape[1], shape[2]), name='average_pooling')(inputs)\n    y_pool = tf.keras.layers.Conv2D(filters=256, kernel_size=1, padding='same', use_bias=False)(y_pool)\n    y_pool = tf.keras.layers.BatchNormalization(name=f'bn_1')(y_pool)\n    y_pool = tf.keras.layers.Activation('relu', name=f'relu_1')(y_pool)\n    y_pool = tf.keras.layers.UpSampling2D((shape[1], shape[2]), interpolation=\"bilinear\")(y_pool)\n\n    y_1 = tf.keras.layers.Conv2D(filters=256, kernel_size=1, dilation_rate=1, padding='same', use_bias=False)(inputs)\n    y_1 = tf.keras.layers.BatchNormalization()(y_1)\n    y_1 = tf.keras.layers.Activation('relu')(y_1)\n\n    y_6 = tf.keras.layers.Conv2D(filters=256, kernel_size=3, dilation_rate=6, padding='same', use_bias=False)(inputs)\n    y_6 = tf.keras.layers.BatchNormalization()(y_6)\n    y_6 = tf.keras.layers.Activation('relu')(y_6)\n\n    y_12 = tf.keras.layers.Conv2D(filters=256, kernel_size=3, dilation_rate=12, padding='same', use_bias=False)(inputs)\n    y_12 = tf.keras.layers.BatchNormalization()(y_12)\n    y_12 = tf.keras.layers.Activation('relu')(y_12)\n\n    y_18 = tf.keras.layers.Conv2D(filters=256, kernel_size=3, dilation_rate=18, padding='same', use_bias=False)(inputs)\n    y_18 = tf.keras.layers.BatchNormalization()(y_18)\n    y_18 = tf.keras.layers.Activation('relu')(y_18)\n\n    y = tf.keras.layers.Concatenate()([y_pool, y_1, y_6, y_12, y_18])\n\n    y = tf.keras.layers.Conv2D(filters=256, kernel_size=1, dilation_rate=1, padding='same', use_bias=False)(y)\n    y = tf.keras.layers.BatchNormalization()(y)\n    y = tf.keras.layers.Activation('relu')(y)\n    return y\n\ndef EfficientAttentionDeepLabV3Plus(num_classes,input_shape=(256,384, 3)):\n    \n    \"\"\" Inputs \"\"\"\n    inputs = tf.keras.layers.Input(input_shape)\n    \n    \"\"\" Pre-trained Output \"\"\"\n    backbone = tf.keras.applications.EfficientNetB0(include_top=False,weights=\"imagenet\",input_tensor=inputs)\n    image_features = backbone.get_layer('block7a_dwconv').output\n    x_a = ASPP(image_features)\n    x_a = tf.keras.layers.UpSampling2D((8, 8), interpolation=\"bilinear\")(x_a)\n\n    \n    \"\"\" Get low-level features \"\"\"\n    x_b = backbone.get_layer('block2a_dwconv').output\n    x_b = self_attention_cnn(x_b)\n    x_b = tf.keras.layers.Conv2D(filters=48, kernel_size=1, padding='same', use_bias=False)(x_b)\n    x_b = tf.keras.layers.BatchNormalization()(x_b)\n    x_b = tf.keras.layers.Activation('relu')(x_b)\n\n    x = tf.keras.layers.Concatenate()([x_a, x_b])\n\n    x = tf.keras.layers.Conv2D(filters=256, kernel_size=3, padding='same', activation='relu',use_bias=False)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n\n    x = tf.keras.layers.Conv2D(filters=256, kernel_size=3, padding='same', activation='relu', use_bias=False)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    x = tf.keras.layers.UpSampling2D((4, 4), interpolation=\"bilinear\")(x)\n\n    \"\"\" Outputs \"\"\"\n    x = tf.keras.layers.Conv2D(num_classes, (1, 1), name='output_layer')(x)\n    x = tf.keras.layers.Activation('sigmoid')(x)\n\n    \"\"\" Model \"\"\"\n    model = tf.keras.models.Model(inputs=inputs, outputs=x)\n    return model\n    \n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:16:58.443103Z","iopub.execute_input":"2023-11-25T13:16:58.443452Z","iopub.status.idle":"2023-11-25T13:16:59.562079Z","shell.execute_reply.started":"2023-11-25T13:16:58.443427Z","shell.execute_reply":"2023-11-25T13:16:59.560978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape=(256,384, 3)\ninputs = tf.keras.layers.Input(input_shape)\nbackbone = tf.keras.applications.EfficientNetB0(include_top=False,weights=\"imagenet\",input_tensor=inputs)\n\n\nimage_features = backbone.get_layer('block7a_dwconv').output\nprint(image_features.shape)\nx_a = ASPP(image_features)\nx_a = tf.keras.layers.UpSampling2D((8, 8), interpolation=\"bilinear\")(x_a)\nprint(x_a.shape)\n\n\"\"\" Get low-level features \"\"\"\nx_b = backbone.get_layer('block2a_dwconv').output\nx_b = self_attention_cnn(x_b)\nprint(x_b.shape)\nx_b = tf.keras.layers.Conv2D(filters=48, kernel_size=1, padding='same', use_bias=False)(x_b)\nx_b = tf.keras.layers.BatchNormalization()(x_b)\nx_b = tf.keras.layers.Activation('relu')(x_b)\nprint(x_b.shape)\nx = tf.keras.layers.Concatenate()([x_a, x_b])\nprint(x.shape)\nx = tf.keras.layers.Conv2D(filters=256, kernel_size=3, padding='same', activation='relu',use_bias=False)(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.Activation('relu')(x)\n\nx = tf.keras.layers.Conv2D(filters=256, kernel_size=3, padding='same', activation='relu', use_bias=False)(x)\nx = tf.keras.layers.BatchNormalization()(x)\nx = tf.keras.layers.Activation('relu')(x)\nprint(x.shape)\nx = tf.keras.layers.UpSampling2D((4, 4), interpolation=\"bilinear\")(x)\n\n\"\"\" Outputs \"\"\"\nx = tf.keras.layers.Conv2D(1, (1, 1), name='output_layer')(x)\nx = tf.keras.layers.Activation('sigmoid')(x)\n\n\"\"\" Model \"\"\"\nmodel = tf.keras.models.Model(inputs=inputs, outputs=x)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:03:16.008846Z","iopub.execute_input":"2023-11-25T13:03:16.009194Z","iopub.status.idle":"2023-11-25T13:03:18.335098Z","shell.execute_reply.started":"2023-11-25T13:03:16.009168Z","shell.execute_reply":"2023-11-25T13:03:18.334154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EfficientAttentionDeepLabV3Plus(num_classes=4,\n                                        input_shape = (R_HEIGHT, R_WIDTH, NUM_CHANNELS))\ntotal_param = format(model.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\nviz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:01:45.451576Z","iopub.execute_input":"2023-11-25T13:01:45.451959Z","iopub.status.idle":"2023-11-25T13:01:49.182073Z","shell.execute_reply.started":"2023-11-25T13:01:45.451923Z","shell.execute_reply":"2023-11-25T13:01:49.181241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_param = format(model.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\nviz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:02:32.436773Z","iopub.execute_input":"2023-11-25T13:02:32.437632Z","iopub.status.idle":"2023-11-25T13:02:49.054780Z","shell.execute_reply.started":"2023-11-25T13:02:32.437596Z","shell.execute_reply":"2023-11-25T13:02:49.053886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DeepLabV3Plus:\n    def __init__(self,\n                 input_shape=(256,384, 3),\n                 num_classes=4,\n                 activation='relu'):\n        self.input_shape = input_shape\n        self.num_classes = num_classes\n        self.activation = activation\n        self.model = self.build_model()\n        self.backbone = tf.keras.applications.EfficientNetB0(include_top=False,\n                                                            weights=\"imagenet\",\n                                                            input_shape=input_shape,\n                                                            )\n        \n        self.encoder = self.backbone.get_layer(\"block7a_dwconv\").output\n        \n        pass","metadata":{"execution":{"iopub.status.busy":"2023-11-25T12:47:22.307747Z","iopub.execute_input":"2023-11-25T12:47:22.308240Z","iopub.status.idle":"2023-11-25T12:47:22.317365Z","shell.execute_reply.started":"2023-11-25T12:47:22.308205Z","shell.execute_reply":"2023-11-25T12:47:22.315894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Custom Loss, metric and Optimizer classes <a class=\"anchor\" id=\"loss_optim\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"code","source":"class DiceScore(tf.keras.metrics.Metric):\n    def __init__(self, name='dice_score', **kwargs):\n        super(DiceScore, self).__init__(name=name, **kwargs)\n        self.intersection = self.add_weight(name='intersection', initializer='zeros')\n        self.union = self.add_weight(name='union', initializer='zeros')\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        y_true = tf.cast(y_true, dtype=tf.float32)\n        y_pred = tf.cast(y_pred, dtype=tf.float32)\n\n        intersection = tf.reduce_sum(y_true * y_pred)\n        union = tf.reduce_sum(y_true) + tf.reduce_sum(y_pred)\n\n        self.intersection.assign_add(intersection)\n        self.union.assign_add(union)\n\n    def result(self):\n        epsilon = tf.keras.backend.epsilon()\n        epsilon = 1\n        dice = (2.0 * self.intersection) / (self.union + epsilon)\n        return dice\n\n    def reset_state(self):\n        self.intersection.assign(0.0)\n        self.union.assign(0.0)\n        \n\n    \nclass IoUScore(tf.keras.metrics.Metric):\n    def __init__(self, name='iou_score', **kwargs):\n        super(IoUScore, self).__init__(name=name, **kwargs)\n        self.intersection = self.add_weight(name='intersection', initializer='zeros')\n        self.union = self.add_weight(name='union', initializer='zeros')\n\n    def update_state(self, y_true, y_pred, sample_weight=None):\n        y_true = tf.cast(y_true, dtype=tf.float32)\n        y_pred = tf.cast(y_pred, dtype=tf.float32)\n\n        intersection = tf.reduce_sum(y_true * y_pred)\n        union = tf.reduce_sum(tf.maximum(y_true, y_pred))\n\n        self.intersection.assign_add(intersection)\n        self.union.assign_add(union)\n\n    def result(self):\n        epsilon = tf.keras.backend.epsilon()\n        epsilon = 1\n        iou = (self.intersection) / (self.union + epsilon)\n        return iou\n\n    def reset_state(self):\n        self.intersection.assign(0.0)\n        self.union.assign(0.0)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:04:42.660614Z","iopub.execute_input":"2023-11-25T13:04:42.661933Z","iopub.status.idle":"2023-11-25T13:04:42.676503Z","shell.execute_reply.started":"2023-11-25T13:04:42.661880Z","shell.execute_reply":"2023-11-25T13:04:42.675486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.array([0., 0., 1., 0.])\ny_true = np.array([[1., 1., 1., 0.]])\n\nd = IoUScore()\nd.update_state(y_true,y_pred)\nprint(d.result().numpy())\n\nd = DiceScore()\nd.update_state(y_true,y_pred)\nprint(d.result().numpy())","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:04:45.388606Z","iopub.execute_input":"2023-11-25T13:04:45.388968Z","iopub.status.idle":"2023-11-25T13:04:45.417380Z","shell.execute_reply.started":"2023-11-25T13:04:45.388939Z","shell.execute_reply":"2023-11-25T13:04:45.416521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DiceLoss(tf.keras.losses.Loss):\n    def __init__(self, from_logits=False, reduction=tf.keras.losses.Reduction.AUTO, name='dice_loss'):\n        super(DiceLoss, self).__init__(reduction=reduction, name=name)\n        self.from_logits = from_logits\n\n    def dice_loss(self, y_true, y_pred):\n        epsilon = tf.keras.backend.epsilon()\n        epsilon = 1\n        intersection = tf.reduce_sum(y_true * y_pred)\n        dice = (2.0 * intersection + epsilon) / (tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) + epsilon)\n        return 1.0 - dice\n\n    def call(self, y_true, y_pred):\n        if self.from_logits:\n            y_pred = tf.nn.sigmoid(y_pred)\n        dice_loss = self.dice_loss(y_true, y_pred)\n        return dice_loss\n    \nclass IoULoss(tf.keras.losses.Loss):\n    def __init__(self, from_logits=False, reduction=tf.keras.losses.Reduction.AUTO, name='iou_loss'):\n        super(IoULoss, self).__init__(reduction=reduction, name=name)\n        self.from_logits = from_logits\n\n    def iou_loss(self, y_true, y_pred):\n        #epsilon = 2*tf.keras.backend.epsilon()\n        epsilon = 1\n        intersection = tf.reduce_sum(y_true * y_pred)\n        union = tf.reduce_sum(tf.maximum(y_true, y_pred))\n        iou = (intersection + epsilon) / (union + epsilon)\n        return 1.0 - iou\n\n    def call(self, y_true, y_pred):\n        if self.from_logits:\n            y_pred = tf.nn.sigmoid(y_pred)\n        iou_loss = self.iou_loss(y_true, y_pred)\n        return iou_loss\n\nclass BCELoss(tf.keras.losses.Loss):\n    def __init__(self, from_logits=False, reduction=tf.keras.losses.Reduction.AUTO, name='bce_loss'):\n        super(BCELoss, self).__init__(reduction=reduction, name=name)\n        self.from_logits = from_logits\n    \n    def bce_loss(self,y_true,y_pred):\n        epsilon = 2*tf.keras.backend.epsilon()\n        y_pred = np.clip(y_pred, epsilon, 1.0 - epsilon)  # Clip to avoid log(0) and log(1)\n        bce_loss = - (y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred))\n        mean_bce_loss = tf.reduce_sum(bce_loss)/len(bce_loss.flatten())\n        return mean_bce_loss\n\n    def call(self, y_true, y_pred):\n        if self.from_logits:\n            y_pred = tf.nn.sigmoid(y_pred)\n        bce_loss = self.bce_loss(y_true, y_pred)\n        return bce_loss\n    \nalpha = 0.8\ngamma = 2.0\n    \nclass FocalLoss(tf.keras.losses.Loss):\n    def __init__(self, from_logits=False, reduction=tf.keras.losses.Reduction.AUTO, name='focal_loss'):\n        super(FocalLoss, self).__init__(reduction=reduction, name=name)\n        self.from_logits = from_logits\n        \n    def focal_loss(self,y_true,y_pred):\n        bce_loss = tf.keras.losses.BinaryCrossentropy(from_logits=self.from_logits,reduction=self.reduction)(y_true, y_pred)\n        pt = np.exp(-bce_loss)\n        focal_loss = np.power(alpha*(1-pt), gamma)*bce_loss\n        return focal_loss\n\n    def call(self, y_true, y_pred):\n        if self.from_logits:\n            y_pred = tf.nn.sigmoid(y_pred)\n        focal_loss = self.focal_loss(y_true,y_pred)\n        return focal_loss\n\n\nclass DiceBCELoss(tf.keras.losses.Loss):\n    def __init__(self, from_logits=False, reduction=tf.keras.losses.Reduction.AUTO, name='dice_bce_loss'):\n        super(DiceBCELoss, self).__init__(reduction=reduction, name=name)\n        self.from_logits = from_logits\n\n    def dice_loss(self, y_true, y_pred):\n        epsilon = tf.keras.backend.epsilon()\n        epsilon = 1\n        intersection = tf.reduce_sum(y_true * y_pred)\n        dice = (2.0 * intersection + epsilon) / (tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) + epsilon)\n        return 1.0 - dice\n\n    def call(self, y_true, y_pred):\n        if self.from_logits:\n            y_pred = tf.nn.sigmoid(y_pred)\n        \n        bce_loss = tf.keras.losses.BinaryCrossentropy(from_logits=self.from_logits, reduction=self.reduction)(y_true, y_pred)\n        dice_loss = self.dice_loss(y_true, y_pred)\n\n        return dice_loss + bce_loss\n    \n\nclass DiceFocalLoss(tf.keras.losses.Loss):\n    def __init__(self, from_logits=False, reduction=tf.keras.losses.Reduction.AUTO, name='dice_focal_loss'):\n        super(DiceFocalLoss, self).__init__(reduction=reduction, name=name)\n        self.from_logits = from_logits\n\n    def dice_loss(self, y_true, y_pred):\n        epsilon = tf.keras.backend.epsilon()\n        epsilon = 1\n        intersection = tf.reduce_sum(y_true * y_pred)\n        dice = (2.0 * intersection + epsilon) / (tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) + epsilon)\n        return 1.0 - dice\n    \n    def focal_loss(self,y_true, y_pred):\n        bce_loss = tf.keras.losses.BinaryCrossentropy(from_logits=self.from_logits,reduction=self.reduction)(y_true, y_pred)\n        pt = np.exp(-bce_loss)\n        focal_loss = np.mean(np.power(alpha*(1-pt), gamma)*bce_loss)\n        return focal_loss\n\n    def call(self, y_true, y_pred):\n        if self.from_logits:\n            y_pred = tf.nn.sigmoid(y_pred)\n        \n        dice_loss = self.dice_loss(y_true, y_pred)\n        focal_loss = self.focal_loss(y_true, y_pred)\n\n        return dice_loss + focal_loss\n    \n    \nclass DiceFocalBCELoss(tf.keras.losses.Loss):\n    def __init__(self, from_logits=False, reduction=tf.keras.losses.Reduction.AUTO, name='dice_focal_bce_loss'):\n        super(DiceFocalBCELoss, self).__init__(reduction=reduction, name=name)\n        self.from_logits = from_logits\n\n    def dice_loss(self, y_true, y_pred):\n        epsilon = tf.keras.backend.epsilon()\n        epsilon = 1\n        intersection = tf.reduce_sum(y_true * y_pred)\n        dice = (2.0 * intersection + epsilon) / (tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) + epsilon)\n        return 1.0 - dice\n    \n    def focal_loss(self,y_true,y_pred):\n        bce_loss = tf.keras.losses.BinaryCrossentropy(from_logits=self.from_logits,reduction=self.reduction)(y_true, y_pred)\n        pt = np.exp(-bce_loss)\n        focal_loss = np.mean(np.power(alpha*(1-pt), gamma)*bce_loss)\n        return focal_loss\n\n    def call(self, y_true, y_pred):\n        if self.from_logits:\n            y_pred = tf.nn.sigmoid(y_pred)\n        \n        bce_loss = tf.keras.losses.BinaryCrossentropy(from_logits=self.from_logits, reduction=self.reduction)(y_true, y_pred)\n        dice_loss = self.dice_loss(y_true, y_pred)\n        focal_loss = self.focal_loss(y_true, y_pred)\n\n        return dice_loss + focal_loss+ bce_loss\n    \n    \n","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:04:47.024251Z","iopub.execute_input":"2023-11-25T13:04:47.024578Z","iopub.status.idle":"2023-11-25T13:04:47.058377Z","shell.execute_reply.started":"2023-11-25T13:04:47.024553Z","shell.execute_reply":"2023-11-25T13:04:47.057394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.array([0., 0., 1., 0.])\ny_true = np.array([[1., 1., 1., 0.]])\n\ncriterion = BCELoss()\nloss = criterion.call(y_true,y_pred)\nprint(f\"{criterion.name} = {loss.numpy()}\")\n\ncriterion = FocalLoss()\nloss = criterion.call(y_true,y_pred)\nprint(f\"{criterion.name} = {loss.numpy()}\")\n\n\ncriterion = DiceLoss()\nloss = criterion.call(y_true,y_pred)\nprint(f\"{criterion.name} = {loss.numpy()}\")\n\ncriterion = IoULoss()\nloss = criterion.call(y_true,y_pred)\nprint(f\"{criterion.name} = {loss.numpy()}\")\n\n\ncriterion = DiceBCELoss()\nloss = criterion.call(y_true,y_pred)\nprint(f\"{criterion.name} = {loss.numpy()}\")\n\n\ncriterion = DiceFocalLoss()\nloss = criterion.call(y_true,y_pred)\nprint(f\"{criterion.name} = {loss.numpy()}\")\n\ncriterion = DiceFocalBCELoss()\nloss = criterion.call(y_true,y_pred)\nprint(f\"{criterion.name} = {loss.numpy()}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:04:48.220828Z","iopub.execute_input":"2023-11-25T13:04:48.221753Z","iopub.status.idle":"2023-11-25T13:04:48.265010Z","shell.execute_reply.started":"2023-11-25T13:04:48.221716Z","shell.execute_reply":"2023-11-25T13:04:48.264096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nfrom wandb.keras import WandbCallback,WandbModelCheckpoint\nfrom kaggle_secrets import UserSecretsClient\n\nuser_secrets = UserSecretsClient()\nwandb_api_key = user_secrets.get_secret(\"wandb_api_key\") \n\nwandb.login(key=wandb_api_key)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:07:55.435868Z","iopub.execute_input":"2023-11-25T13:07:55.436217Z","iopub.status.idle":"2023-11-25T13:07:58.333499Z","shell.execute_reply.started":"2023-11-25T13:07:55.436192Z","shell.execute_reply":"2023-11-25T13:07:58.332370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = dict(\n    input_shape = (R_HEIGHT, R_WIDTH, NUM_CHANNELS),\n    batch_size = BATCH_SIZE,\n    optimizer = tf.keras.optimizers.Adam(),\n    loss = DiceBCELoss(),\n    metrics = [DiceScore(), IoUScore()],\n    epochs = 1,\n)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:18:14.531209Z","iopub.execute_input":"2023-11-25T13:18:14.531558Z","iopub.status.idle":"2023-11-25T13:18:15.589789Z","shell.execute_reply.started":"2023-11-25T13:18:14.531530Z","shell.execute_reply":"2023-11-25T13:18:15.588711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = wandb.init(entity='taki',\n                 project = 'cloud_formation_segmentation',\n                 name = 'test_1e_Effiecientnetb2AttentionDeeplabV3Plus',\n                 config = config,\n                 save_code = True,\n                )","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:05:42.931566Z","iopub.execute_input":"2023-11-25T13:05:42.932398Z","iopub.status.idle":"2023-11-25T13:06:15.393615Z","shell.execute_reply.started":"2023-11-25T13:05:42.932360Z","shell.execute_reply":"2023-11-25T13:06:15.392614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining Callbacks <a class=\"anchor\" id=\"model_callbacks\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"code","source":"work_dir = \"/kaggle/working/\"\nmodel_name = \"cloud_1e_Effiecientnetb2AttentionDeeplabV3Plus\"\n\n\nwandb_callback = WandbCallback(save_model=False)\nwandb_checkpoint = WandbModelCheckpoint(filepath=\"weights-improvment-{epoch:02d}-{val_dice_score:.4f}.h5\",\n                                        monitor='val_dice_score',\n                                        mode=\"max\",\n                                        save_best_only=True,\n                                        save_freq=\"epoch\",\n                                        period=5,)\n\nreduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_dice_score\",\n                                                 mode=\"max\",\n                                                 factor=0.5,\n                                                 min_lr=1e-6,\n                                                 patience=1,\n                                                 verbose=0)\n\nearly_stop = tf.keras.callbacks.EarlyStopping(monitor=\"val_dice_score\",\n                                              mode=\"max\",\n                                              patience=3,\n                                              verbose=0,\n                                              restore_best_weights=True)\n\n\nckpt_filepath = os.path.join(work_dir,model_name,\"ckpts\",\"weights-improvement-{epoch:02d}-{val_loss:.4f}.hdf5\")\n\nmodel_checkpoint = tf.keras.callbacks.ModelCheckpoint(ckpt_filepath,\n                                                      monitor=\"val_loss\",\n                                                      mode=\"min\",\n                                                      save_best_only=False,\n                                                      save_freq=\"epoch\",\n                                                      period=2,\n                                                      )\n\ncsv_filepath = os.path.join(work_dir,model_name,\"logs\",\"training.csv\")\ncsv_log = tf.keras.callbacks.CSVLogger(csv_filepath)\n\ncallbacks = [reduce_lr,early_stop,wandb_callback, wandb_checkpoint]","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:08:05.838095Z","iopub.execute_input":"2023-11-25T13:08:05.838935Z","iopub.status.idle":"2023-11-25T13:08:06.808100Z","shell.execute_reply.started":"2023-11-25T13:08:05.838877Z","shell.execute_reply":"2023-11-25T13:08:06.807097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the Model <a class=\"anchor\" id=\"model_train\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = sm.Unet('efficientnetb2',\n                input_shape=config[\"input_shape\"],\n                classes=4,\n                activation='sigmoid',\n                encoder_weights='imagenet')\n\n\nmodel.compile(optimizer=config['optimizer'], loss=config['loss'], metrics=config['metrics'])\n\ntotal_param = format(model.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\nprint(model.name)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:16:08.411290Z","iopub.execute_input":"2023-11-25T13:16:08.411698Z","iopub.status.idle":"2023-11-25T13:16:13.708696Z","shell.execute_reply.started":"2023-11-25T13:16:08.411660Z","shell.execute_reply":"2023-11-25T13:16:13.707435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()\n\nmodel = EfficientAttentionDeepLabV3Plus(num_classes=4,\n                                        input_shape = (R_HEIGHT, R_WIDTH, NUM_CHANNELS))\n\n\nmodel.compile(optimizer=config['optimizer'], loss=config['loss'], metrics=config['metrics'])\n\n\ntotal_param = format(model.count_params(),\",\")\nprint(f\"Total no of parameters = {total_param}\")\n#viz_model(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:22:29.959112Z","iopub.execute_input":"2023-11-25T13:22:29.959438Z","iopub.status.idle":"2023-11-25T13:22:33.707111Z","shell.execute_reply.started":"2023-11-25T13:22:29.959413Z","shell.execute_reply":"2023-11-25T13:22:33.706101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test everything before training the model\nprint(model.input_shape,'\\t',train_generator.image_shape,'\\t',val_generator.image_shape)\nprint(\"GPU available!\" if tf.test.is_gpu_available() else \"GPU is not available\")\nprint(model.optimizer)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:22:33.709295Z","iopub.execute_input":"2023-11-25T13:22:33.709689Z","iopub.status.idle":"2023-11-25T13:22:34.640087Z","shell.execute_reply.started":"2023-11-25T13:22:33.709649Z","shell.execute_reply":"2023-11-25T13:22:34.639102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run this line if you used dropout layer or segmentation_model library\n# tf.config.optimizer.set_experimental_options({\"layout_optimizer\": False}) ","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:22:34.641567Z","iopub.execute_input":"2023-11-25T13:22:34.641932Z","iopub.status.idle":"2023-11-25T13:22:35.751950Z","shell.execute_reply.started":"2023-11-25T13:22:34.641878Z","shell.execute_reply":"2023-11-25T13:22:35.750938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import multiprocessing\nnum_cores = multiprocessing.cpu_count()\nprint(num_cores)","metadata":{"execution":{"iopub.status.busy":"2023-11-25T13:22:35.754060Z","iopub.execute_input":"2023-11-25T13:22:35.754407Z","iopub.status.idle":"2023-11-25T13:22:36.766194Z","shell.execute_reply.started":"2023-11-25T13:22:35.754373Z","shell.execute_reply":"2023-11-25T13:22:36.765047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.perf_counter()\n\nhistory = model.fit(train_generator,\n                    validation_data=val_generator,\n                    epochs=config['epochs'],\n                    callbacks=callbacks,\n                    workers=num_cores,\n                    use_multiprocessing=False\n                    )\n\nend = time.perf_counter()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Error** - `E tensorflow/core/grappler/optimizers/meta_optimizer.cc:954] layout failed: INVALID_ARGUMENT: Size of values 0 does not match size of permutation 4 @ fanin shape inmodel_1/dropout_9/dropout/SelectV2-2-TransposeNHWCToNCHW-LayoutOptimizer`\n\n**Reason** - `Dropout layer`\n\n**Solution** - `1. tf.config.optimizer.set_experimental_options({\"layout_optimizer\": False})`\n               `2. Remove dropout layer`","metadata":{"execution":{"iopub.status.busy":"2023-06-10T12:27:08.580742Z","iopub.execute_input":"2023-06-10T12:27:08.581130Z","iopub.status.idle":"2023-06-10T12:27:08.586615Z","shell.execute_reply.started":"2023-06-10T12:27:08.581098Z","shell.execute_reply":"2023-06-10T12:27:08.585686Z"}}},{"cell_type":"code","source":"print(f\"Total training time = {(end-start)/3600})","metadata":{"execution":{"iopub.status.busy":"2023-07-27T01:48:52.064342Z","iopub.execute_input":"2023-07-27T01:48:52.065122Z","iopub.status.idle":"2023-07-27T01:48:55.622326Z","shell.execute_reply.started":"2023-07-27T01:48:52.065051Z","shell.execute_reply":"2023-07-27T01:48:55.621215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_accuracy_loss(history):\n    f,ax = plt.subplots(1,3,figsize=(16,4))\n    ax = ax.ravel()\n    \n    ax[0].plot([None]+history.history['loss'],'o-')\n    ax[0].plot([None]+history.history['val_loss'],'o-')\n    ax[0].legend(['Train Loss','Validation Loss'],loc = 0)\n    ax[0].set_title('Training & Validation Loss')\n    ax[0].set_xlabel('Epoch')\n    ax[0].set_ylabel('Loss')\n    ax[0].grid(True)\n\n    ax[1].plot([None]+history.history['dice_score'],'o-')\n    ax[1].plot([None]+history.history['val_dice_score'],'o-')\n    ax[1].legend(['Training Dice','Validation Dice'],loc = 0)\n    ax[1].set_title('Training & Validation Dice Score')\n    ax[1].set_xlabel('Epoch')\n    ax[1].set_ylabel('Dice Score')\n    ax[1].grid(True)\n    \n    ax[2].plot([None]+history.history['iou_score'],'o-')\n    ax[2].plot([None]+history.history['val_iou_score'],'o-')\n    ax[2].legend(['Train IOU','Validation IOU'],loc = 0)\n    ax[2].set_title('Training & Validation IOU Score')\n    ax[2].set_xlabel('Epoch')\n    ax[2].set_ylabel('IOU Score')\n    ax[2].grid(True)\n    \n    #plt.style.use('ggplot')\n    plt.tight_layout()\n    plt.show()\n    \n    return None\n","metadata":{"execution":{"iopub.status.busy":"2023-07-27T01:48:55.623851Z","iopub.execute_input":"2023-07-27T01:48:55.624559Z","iopub.status.idle":"2023-07-27T01:48:58.841902Z","shell.execute_reply.started":"2023-07-27T01:48:55.624525Z","shell.execute_reply":"2023-07-27T01:48:58.840159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_accuracy_loss(history)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T01:48:58.844706Z","iopub.execute_input":"2023-07-27T01:48:58.845292Z","iopub.status.idle":"2023-07-27T01:49:03.112772Z","shell.execute_reply.started":"2023-07-27T01:48:58.845257Z","shell.execute_reply":"2023-07-27T01:49:03.111388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saving and Downloading the trained Model <a class=\"anchor\" id=\"model_save\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"code","source":"work_dir = \"/kaggle/working/\"\nfilepath = os.path.join(work_dir,model_name,f\"{model_name}.h5\")\nmodel_path = model.save(filepath,save_format=\"h5\")\nwandb.save(filepath)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T01:49:03.118018Z","iopub.execute_input":"2023-07-27T01:49:03.120840Z","iopub.status.idle":"2023-07-27T01:49:07.408110Z","shell.execute_reply.started":"2023-07-27T01:49:03.120803Z","shell.execute_reply":"2023-07-27T01:49:07.406860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2023-07-27T01:49:07.409684Z","iopub.execute_input":"2023-07-27T01:49:07.410394Z","iopub.status.idle":"2023-07-27T01:49:19.904992Z","shell.execute_reply.started":"2023-07-27T01:49:07.410361Z","shell.execute_reply":"2023-07-27T01:49:19.904131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport subprocess\nfrom IPython.display import FileLink, display\n\ndef download_file(path, download_file_name):\n    os.chdir('/kaggle/working/')\n    zip_name = f\"/kaggle/working/{download_file_name}.zip\"\n    command = f\"zip {zip_name} {path} -r\"\n    result = subprocess.run(command, shell=True, capture_output=True, text=True)\n    if result.returncode != 0:\n        print(\"Unable to run zip command!\")\n        print(result.stderr)\n        return\n    display(FileLink(f'{download_file_name}.zip'))\n","metadata":{"execution":{"iopub.status.busy":"2023-07-04T22:47:13.038954Z","iopub.execute_input":"2023-07-04T22:47:13.039642Z","iopub.status.idle":"2023-07-04T22:47:13.051281Z","shell.execute_reply.started":"2023-07-04T22:47:13.039590Z","shell.execute_reply":"2023-07-04T22:47:13.050261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"download_file(filepath, \"cloud_model_20_effiUnet\")","metadata":{"execution":{"iopub.status.busy":"2023-07-04T22:47:13.053152Z","iopub.execute_input":"2023-07-04T22:47:13.053932Z","iopub.status.idle":"2023-07-04T22:47:21.998651Z","shell.execute_reply.started":"2023-07-04T22:47:13.053894Z","shell.execute_reply":"2023-07-04T22:47:21.997646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate model on Test Set and make submission <a class=\"anchor\" id=\"model_eval_test\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"code","source":"work_dir = \"/kaggle/working/\"\nfilepath = os.path.join(work_dir,f\"{model_name}.h5\")\nfile = wandb.restore(f\"{model_name}.h5\",run_path='taki/cloud_formation_segmentation/runs/gcdsx2sl')\nmodel = tf.keras.saving.load_model(filepath,compile=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T08:38:29.321695Z","iopub.execute_input":"2023-07-27T08:38:29.322077Z","iopub.status.idle":"2023-07-27T08:38:32.380803Z","shell.execute_reply.started":"2023-07-27T08:38:29.322018Z","shell.execute_reply":"2023-07-27T08:38:32.379763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`Classwise best threshold and minsize params: {0: (0.5, 25000), 1: (0.6, 21000), 2: (0.3, 20000), 3: (0.5, 10000)}` `Collected from` https://www.kaggle.com/code/artgor/classification-in-catalyst-with-utility-scripts#Post-processing","metadata":{}},{"cell_type":"code","source":"#minsizes = [20000 ,20000, 22500, 10000]\nminsizes = [25000 ,21000, 2000, 10000]\nsigmoid = lambda x: 1 / (1 + np.exp(-x))\nthresholds = [0.5, 0.6, 0.3, 0.5]\n\ndef mask2rle(img):\n    '''\n    Convert mask to rle.\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels= img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n\ndef post_process(probability, threshold, min_size):\n    \"\"\"\n    Post processing of each predicted mask, components with lesser number of pixels\n    than `min_size` are ignored\n    \"\"\"\n    \n    mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]\n    \n    num_component, component = cv2.connectedComponents(mask.astype(np.uint8))\n    predictions = np.zeros((350, 525), np.float32)\n    num = 0\n    for c in range(1, num_component):\n        p = (component == c)\n        if p.sum() > min_size:\n            predictions[p] = 1\n            num += 1\n    return predictions, num\n            ","metadata":{"execution":{"iopub.status.busy":"2023-07-27T08:39:57.931747Z","iopub.execute_input":"2023-07-27T08:39:57.932157Z","iopub.status.idle":"2023-07-27T08:39:57.943215Z","shell.execute_reply.started":"2023-07-27T08:39:57.932123Z","shell.execute_reply":"2023-07-27T08:39:57.942230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://www.kaggle.com/code/ratthachat/cloud-convexhull-polygon-postprocessing-no-gpu?scriptVersionId=20977692","metadata":{}},{"cell_type":"code","source":"def draw_convex_hull(mask, mode='approx'):\n    \n    img = np.zeros(mask.shape)\n    contours, hier = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n    \n    for c in contours:\n        if mode=='rect': # simple rectangle\n            x, y, w, h = cv2.boundingRect(c)\n            cv2.rectangle(img, (x, y), (x+w, y+h), (255, 255, 255), -1)\n        elif mode=='convex': # minimum convex hull\n            hull = cv2.convexHull(c)\n            cv2.drawContours(img, [hull], 0, (255, 255, 255),-1)\n        elif mode=='approx':\n            epsilon = 0.02*cv2.arcLength(c,True)\n            approx = cv2.approxPolyDP(c,epsilon,True)\n            cv2.drawContours(img, [approx], 0, (255, 255, 255),-1)\n        else: # minimum area rectangle\n            rect = cv2.minAreaRect(c)\n            box = cv2.boxPoints(rect)\n            box = np.int0(box)\n            cv2.drawContours(img, [box], 0, (255, 255, 255),-1)\n    return img/255.","metadata":{"execution":{"iopub.status.busy":"2023-07-27T08:40:01.425902Z","iopub.execute_input":"2023-07-27T08:40:01.426260Z","iopub.status.idle":"2023-07-27T08:40:01.435738Z","shell.execute_reply.started":"2023-07-27T08:40:01.426231Z","shell.execute_reply":"2023-07-27T08:40:01.434513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_images = test_generator.__getitem__(0)\nlow = index * batch_Size\nhigh = min(low + batch_Size, total_samples)\nbatch_indexes = list(range(low,high))\nbatch_indexes_sub = range(low*4, high*4)\nbatch_size = batch_images.shape[0]\nbatch_predicted_masks = model.predict(batch_images)\nbatch_predicted_masks = batch_predicted_masks.round()\nbatch_predicted_masks[0]\nresized_batch_predicted_masks = np.zeros((32, 350, 525, 4))\nfor i in range(batch_predicted_masks.shape[0]):\n    resized_batch_predicted_masks[i] = cv2.resize(batch_predicted_masks[i], (525,350))\nresized_batch_predicted_masks = resized_batch_predicted_masks.astype(np.int64)\n#print(resized_batch_predicted_masks[0]) ","metadata":{"execution":{"iopub.status.busy":"2023-07-27T10:33:10.621603Z","iopub.execute_input":"2023-07-27T10:33:10.621991Z","iopub.status.idle":"2023-07-27T10:33:14.093543Z","shell.execute_reply.started":"2023-07-27T10:33:10.621962Z","shell.execute_reply":"2023-07-27T10:33:14.092492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv(\"/kaggle/input/understanding_cloud_organization/sample_submission.csv\")\n\ntotal_samples = test_generator.total_samples\nbatch_Size = test_generator.batch_size\nno_batches = test_generator.__len__() \n\nfor index in range(no_batches):\n    batch_images = test_generator.__getitem__(index)\n    low = index * batch_Size\n    high = min(low + batch_Size, total_samples)\n    batch_indexes = list(range(low,high))\n    batch_indexes_sub = range(low*4, high*4)\n    batch_size = batch_images.shape[0]\n    batch_predicted_masks = model.predict(batch_images) \n    batch_predicted_masks = batch_predicted_masks.round()\n\n    resized_batch_predicted_masks = np.zeros((32, 350, 525, 4))\n    for i in range(batch_predicted_masks.shape[0]):\n        resized_batch_predicted_masks[i] = cv2.resize(batch_predicted_masks[i], (525,350))\n    #resized_batch_predicted_masks = resized_batch_predicted_masks.astype(np.int32)\n    \n    for k in batch_indexes_sub:\n        m = k%32\n        if m<32:\n            mask, num_predict = post_process(resized_batch_predicted_masks[m][:,:,m%4], thresholds[m%4], minsizes[m%4])\n            mask = draw_convex_hull(mask.astype(np.uint8))\n            rle = mask2rle(mask)\n            sub_df.loc[k][\"EncodedPixels\"] = rle\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-27T11:02:26.727972Z","iopub.execute_input":"2023-07-27T11:02:26.728365Z","iopub.status.idle":"2023-07-27T11:09:33.900859Z","shell.execute_reply.started":"2023-07-27T11:02:26.728335Z","shell.execute_reply":"2023-07-27T11:09:33.899810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_indexes_sub","metadata":{"execution":{"iopub.status.busy":"2023-07-27T11:12:43.721051Z","iopub.execute_input":"2023-07-27T11:12:43.721601Z","iopub.status.idle":"2023-07-27T11:12:43.730021Z","shell.execute_reply.started":"2023-07-27T11:12:43.721523Z","shell.execute_reply":"2023-07-27T11:12:43.728943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.head(30)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T11:12:46.976635Z","iopub.execute_input":"2023-07-27T11:12:46.976994Z","iopub.status.idle":"2023-07-27T11:12:46.989946Z","shell.execute_reply.started":"2023-07-27T11:12:46.976967Z","shell.execute_reply":"2023-07-27T11:12:46.988879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.loc[10]","metadata":{"execution":{"iopub.status.busy":"2023-07-27T11:12:54.803359Z","iopub.execute_input":"2023-07-27T11:12:54.804340Z","iopub.status.idle":"2023-07-27T11:12:54.811882Z","shell.execute_reply.started":"2023-07-27T11:12:54.804289Z","shell.execute_reply":"2023-07-27T11:12:54.810922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T11:13:13.137465Z","iopub.execute_input":"2023-07-27T11:13:13.138660Z","iopub.status.idle":"2023-07-27T11:13:14.182732Z","shell.execute_reply.started":"2023-07-27T11:13:13.138618Z","shell.execute_reply":"2023-07-27T11:13:14.181402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring Predicticted Mask on Validation Set <a class=\"anchor\" id=\"model_eval_val\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"code","source":"def batch_post_process(batch_pred_masks,\n                       thresholds=[0.5, 0.6, 0.3, 0.5],\n                       minsizes=[20000 ,20000, 22500, 10000]):\n    \n    batch_processed_masks = np.zeros(batch_pred_masks.shape)\n    for k in range(batch_pred_masks.shape[0]):\n        for i in range(4):\n            probability = batch_pred_masks[k][:,:,i]\n            min_size = minsizes[i]\n            threshold = thresholds[i]\n\n            mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]\n            num_component, component = cv2.connectedComponents(mask.astype(np.uint8))\n            predictions = np.zeros((R_HEIGHT, R_WIDTH), np.float32)\n            num = 0\n            for c in range(1, num_component):\n                p = (component == c)\n                if p.sum() > min_size:\n                    predictions[p] = 1\n                    num += 1\n            mask = predictions\n            mask = draw_convex_hull(mask.astype(np.uint8))\n            batch_processed_masks[k][:,:,i] = mask.astype(np.uint8)\n    return batch_processed_masks","metadata":{"execution":{"iopub.status.busy":"2023-07-27T10:56:28.847139Z","iopub.execute_input":"2023-07-27T10:56:28.847799Z","iopub.status.idle":"2023-07-27T10:56:28.858461Z","shell.execute_reply.started":"2023-07-27T10:56:28.847766Z","shell.execute_reply":"2023-07-27T10:56:28.857444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_generator =  DataGenerator(dataframe=df_val,\n                               root_dir=train_image_path,\n                               mode=\"fit\",\n                               resize=True,\n                               resize_width=R_WIDTH,\n                               resize_height=R_HEIGHT,\n                               num_channels=NUM_CHANNELS,\n                               batch_size=32,\n                               augmentations=None)","metadata":{"execution":{"iopub.status.busy":"2023-07-04T23:31:21.624473Z","iopub.execute_input":"2023-07-04T23:31:21.625480Z","iopub.status.idle":"2023-07-04T23:31:21.631275Z","shell.execute_reply.started":"2023-07-04T23:31:21.625443Z","shell.execute_reply":"2023-07-04T23:31:21.630006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = 2\nbatch_images,batch_masks = val_generator.__getitem__(index)\nbatch_predicted_masks = model.predict(batch_images)\nbatch_predicted_masks = batch_predicted_masks.round()\nprocessed_batch_predicted_masks = batch_post_process(batch_predicted_masks)\nprint(batch_images.shape)\nprint(batch_masks.shape)\nprint(batch_predicted_masks.shape)\nprint(processed_batch_predicted_masks.shape)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T10:57:16.994203Z","iopub.execute_input":"2023-07-27T10:57:16.994611Z","iopub.status.idle":"2023-07-27T10:57:18.706072Z","shell.execute_reply.started":"2023-07-27T10:57:16.994554Z","shell.execute_reply":"2023-07-27T10:57:18.705276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for img,masks,pred_masks in zip(batch_images[3:6],batch_masks[3:6],processed_batch_predicted_masks[3:6]):\n    masks = (masks[:,:,0], masks[:,:,1],masks[:,:,2],masks[:,:,3])\n    pred_masks = (pred_masks[:,:,0], pred_masks[:,:,1],pred_masks[:,:,2],pred_masks[:,:,3])\n    print(\"Image, Masks and Predicted Masks\")\n    show_img_with_masks(img,masks,comment=\"(ground truth)\")\n    show_img_with_masks(img,pred_masks,comment=\"(predicted)\")\n     ","metadata":{"execution":{"iopub.status.busy":"2023-07-27T10:57:18.708000Z","iopub.execute_input":"2023-07-27T10:57:18.708654Z","iopub.status.idle":"2023-07-27T10:57:30.996503Z","shell.execute_reply.started":"2023-07-27T10:57:18.708616Z","shell.execute_reply":"2023-07-27T10:57:30.995476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Something is wrong in post processing. Couldn't figure out! probably sigmoid function","metadata":{"execution":{"iopub.status.busy":"2023-07-27T10:58:08.016704Z","iopub.execute_input":"2023-07-27T10:58:08.017083Z","iopub.status.idle":"2023-07-27T10:58:08.021546Z","shell.execute_reply.started":"2023-07-27T10:58:08.017053Z","shell.execute_reply":"2023-07-27T10:58:08.020617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Acknowledgements <a class=\"anchor\" id=\"ack\"></a>\n[Go back to the Table of Contents](#contents) <br>","metadata":{}},{"cell_type":"markdown","source":"* [Satellite Clouds: U-Net with ResNet Encoder](https://www.kaggle.com/code/xhlulu/satellite-clouds-u-net-with-resnet-encoder)\n* [Cloud: ConvexHull& Polygon PostProcessing (No GPU)](https://www.kaggle.com/code/ratthachat/cloud-convexhull-polygon-postprocessing-no-gpu?scriptVersionId=20977692)\n* [Jupyter Notebook Tricks](https://www.kaggle.com/code/tientd95/jupyter-notebook-tricks)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}