مرحبا بكم جميعا
","metadata":{}},{"cell_type":"markdown","source":"\n#تحميل المكتابات المطلوبه
","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport json\n\nimport cv2\n\nfrom tensorflow import keras\nimport tensorflow as tf\nimport keras\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.layers import Input\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.losses import binary_crossentropy\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.models import load_model\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:03.014184Z","iopub.execute_input":"2021-11-15T21:36:03.014979Z","iopub.status.idle":"2021-11-15T21:36:03.881540Z","shell.execute_reply.started":"2021-11-15T21:36:03.014944Z","shell.execute_reply":"2021-11-15T21:36:03.880534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#قراءه البيانات
\n","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/sartorius-cell-instance-segmentation/train.csv')\nprint(train_df.shape)\ntrain_df.head(4)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:03.883860Z","iopub.execute_input":"2021-11-15T21:36:03.884239Z","iopub.status.idle":"2021-11-15T21:36:04.490213Z","shell.execute_reply.started":"2021-11-15T21:36:03.884194Z","shell.execute_reply":"2021-11-15T21:36:04.489165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=train_df.head(n=20000)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:04.492341Z","iopub.execute_input":"2021-11-15T21:36:04.492931Z","iopub.status.idle":"2021-11-15T21:36:04.519491Z","shell.execute_reply.started":"2021-11-15T21:36:04.492884Z","shell.execute_reply":"2021-11-15T21:36:04.518292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_decode(mask_rle, shape, color=1):\n '''\n mask_rle: run-length as string formated (start length)\n shape: (height,width) of array to return \n Returns numpy array, 1 - mask, 0 - background\n\n '''\n s = mask_rle.split()\n starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n starts -= 1\n ends = starts + lengths\n img = np.zeros((shape[0] * shape[1], shape[2]), dtype=np.float32)\n for lo, hi in zip(starts, ends):\n img[lo : hi] = color\n return img.reshape(shape)\n\n\ndef build_masks(labels,input_shape, colors=True):\n height, width = input_shape\n if colors:\n mask = np.zeros((height, width, 3))\n for label in labels:\n mask += rle_decode(label, shape=(height,width , 3), color=np.random.rand(3))\n else:\n mask = np.zeros((height, width, 1))\n for label in labels:\n mask += rle_decode(label, shape=(height, width, 1))\n mask = mask.clip(0, 1)\n return mask\n\ndef rle2maskResize(rle):\n # CONVERT RLE TO MASK \n if (len(rle)==0): \n return np.zeros((256,256) ,dtype=np.uint8)\n \n height= 520\n width = 704\n mask= np.zeros( width*height ,dtype=np.uint8)\n\n array = np.asarray([int(x) for x in rle.split()])\n starts = array[0::2]-1\n lengths = array[1::2] \n for index, start in enumerate(starts):\n mask[int(start):int(start+lengths[index])] = 1\n \n return mask.reshape( (height,width), order='F' )[::2,::2]","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:04.522243Z","iopub.execute_input":"2021-11-15T21:36:04.522753Z","iopub.status.idle":"2021-11-15T21:36:04.540816Z","shell.execute_reply.started":"2021-11-15T21:36:04.522706Z","shell.execute_reply":"2021-11-15T21:36:04.539510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#عرض الصوره
","metadata":{}},{"cell_type":"code","source":"sample_filename = '0030fd0e6378'\nsample_image_df = train_df[train_df['id'] == sample_filename]\nsample_path = f\"../input/sartorius-cell-instance-segmentation/train/{sample_image_df['id'].iloc[0]}.png\"\nsample_img = cv2.imread(sample_path)\nsample_rles = sample_image_df['annotation'].values\n\nsample_masks1=build_masks(sample_rles,input_shape=(520, 704), colors=False)\nsample_masks2=build_masks(sample_rles,input_shape=(520, 704), colors=False)\n\nfig, axs = plt.subplots(3, figsize=(20, 20))\naxs[0].imshow(sample_img)\naxs[0].axis('off')\n\naxs[1].imshow(sample_masks1)\naxs[1].axis('off')\n\naxs[2].imshow(sample_masks2)\naxs[2].axis('off')","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:04.542918Z","iopub.execute_input":"2021-11-15T21:36:04.543308Z","iopub.status.idle":"2021-11-15T21:36:06.042177Z","shell.execute_reply.started":"2021-11-15T21:36:04.543224Z","shell.execute_reply":"2021-11-15T21:36:06.041072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataGenerator(tf.keras.utils.Sequence):\n 'Generates data for Keras'\n def __init__(self, list_IDs, df, target_df=None, mode='fit',\n base_path='../input/sartorius-cell-instance-segmentation/train',\n batch_size=32, dim=(256, 256), n_channels=3,\n n_classes=3, random_state=2019, shuffle=True):\n self.dim = dim\n self.batch_size = batch_size\n self.df = df\n self.mode = mode\n self.base_path = base_path\n self.target_df = target_df\n self.list_IDs = list_IDs\n self.n_channels = n_channels\n self.n_classes = n_classes\n self.shuffle = shuffle\n self.random_state = random_state\n \n self.on_epoch_end()\n\n def __len__(self):\n 'Denotes the number of batches per epoch'\n return int(np.floor(len(self.list_IDs) / self.batch_size))\n\n def __getitem__(self, index):\n 'Generate one batch of data'\n # Generate indexes of the batch\n indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n # Find list of IDs\n list_IDs_batch = [self.list_IDs[k] for k in indexes]\n \n X = self.__generate_X(list_IDs_batch)\n \n if self.mode == 'fit':\n y = self.__generate_y(list_IDs_batch)\n return X, y\n \n elif self.mode == 'predict':\n return X\n\n else:\n raise AttributeError('The mode parameter should be set to \"fit\" or \"predict\".')\n \n def on_epoch_end(self):\n 'Updates indexes after each epoch'\n self.indexes = np.arange(len(self.list_IDs))\n if self.shuffle == True:\n np.random.seed(self.random_state)\n np.random.shuffle(self.indexes)\n \n def __generate_X(self, list_IDs_batch):\n 'Generates data containing batch_size samples'\n # Initialization\n X = np.empty((self.batch_size, *self.dim, self.n_channels))\n \n # Generate data\n for i, ID in enumerate(list_IDs_batch):\n im_name = self.df['id'].iloc[ID]\n img_path = f\"{self.base_path}/{im_name}.png\"\n img = self.__load_grayscale(img_path)\n \n \n # Store samples\n X[i,] = img \n\n return X\n \n def __generate_y(self, list_IDs_batch):\n y = np.empty((self.batch_size, *self.dim, self.n_classes), dtype=int)\n \n for i, ID in enumerate(list_IDs_batch):\n im_name = self.df['id'].iloc[ID]\n image_df = self.target_df[self.target_df['id'] == im_name]\n \n rles = image_df['annotation'].values\n masks = build_masks(rles,(520,704), colors=False)\n masks = cv2.resize(masks, (256, 256))\n #masks=masks.transpose(1,0)\n masks=np.expand_dims(masks, axis=-1)\n y[i, ] = masks\n\n return y\n \n def __load_grayscale(self, img_path):\n img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)\n # resize image\n dsize = (256, 256)\n img = cv2.resize(img, dsize)\n \n img = img.astype(np.float32) / 255.\n img = np.expand_dims(img, axis=-1)\n\n return img\n \n def __load_rgb(self, img_path):\n img = cv2.imread(img_path)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n img = img.astype(np.float32) / 255.\n\n return img","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:06.043416Z","iopub.execute_input":"2021-11-15T21:36:06.043678Z","iopub.status.idle":"2021-11-15T21:36:06.434375Z","shell.execute_reply.started":"2021-11-15T21:36:06.043641Z","shell.execute_reply":"2021-11-15T21:36:06.433345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 16\n\ntrain_idx, val_idx = train_test_split(\n train_df.index, random_state=2019, test_size=0.2 # mask_count_df\n)\ntrain_generator = DataGenerator(\n train_idx, \n df=train_df,\n target_df=train_df,\n batch_size=BATCH_SIZE, \n n_classes=3\n)\nval_generator = DataGenerator(\n val_idx, \n df=train_df,\n target_df=train_df,\n batch_size=BATCH_SIZE, \n n_classes=3\n)\n\nplt.figure(figsize=(10,10))\nfor i in range(1):\n images, mask = val_generator[i]\n print(\"Dimension of image:\", images.shape)\n print(\"Dimension of mask:\", mask.shape)\n plt.imshow(images[0,:,:,0], cmap=\"gray\")\n plt.imshow(mask[0,:,:,0], alpha=0.6, cmap=\"Blues\")\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:06.436715Z","iopub.execute_input":"2021-11-15T21:36:06.437552Z","iopub.status.idle":"2021-11-15T21:36:09.810264Z","shell.execute_reply.started":"2021-11-15T21:36:06.437503Z","shell.execute_reply":"2021-11-15T21:36:09.809242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#استخدام ال (U-NeT)
","metadata":{}},{"cell_type":"code","source":"from keras import backend as K\nfrom keras.losses import binary_crossentropy\nimport tensorflow as tf\n\ndef dice_coef(y_true, y_pred, smooth=1):\n y_true_f = K.flatten(y_true)\n y_pred_f = K.flatten(y_pred)\n intersection = K.sum(y_true_f * y_pred_f)\n return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n\ndef iou_coef(y_true, y_pred, smooth=1):\n intersection = K.sum(K.abs(y_true * y_pred), axis=[1,2,3])\n union = K.sum(y_true,[1,2,3])+K.sum(y_pred,[1,2,3])-intersection\n iou = K.mean((intersection + smooth) / (union + smooth), axis=0)\n return iou\n\ndef dice_loss(y_true, y_pred):\n smooth = 1.\n y_true_f = K.flatten(y_true)\n y_pred_f = K.flatten(y_pred)\n intersection = y_true_f * y_pred_f\n score = (2. * K.sum(intersection) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n return 1. - score\n\ndef bce_dice_loss(y_true, y_pred):\n return binary_crossentropy(tf.cast(y_true, tf.float32), y_pred) + 0.5 * dice_loss(tf.cast(y_true, tf.float32), y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:09.811854Z","iopub.execute_input":"2021-11-15T21:36:09.812722Z","iopub.status.idle":"2021-11-15T21:36:09.829709Z","shell.execute_reply.started":"2021-11-15T21:36:09.812680Z","shell.execute_reply":"2021-11-15T21:36:09.828630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## تحميل ال (U-Net)","metadata":{}},{"cell_type":"code","source":"! pip install segmentation-models","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:09.831470Z","iopub.execute_input":"2021-11-15T21:36:09.832109Z","iopub.status.idle":"2021-11-15T21:36:22.248848Z","shell.execute_reply.started":"2021-11-15T21:36:09.832064Z","shell.execute_reply":"2021-11-15T21:36:22.247642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install git+https://github.com/qubvel/segmentation_models","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:22.254119Z","iopub.execute_input":"2021-11-15T21:36:22.255061Z","iopub.status.idle":"2021-11-15T21:36:34.975347Z","shell.execute_reply.started":"2021-11-15T21:36:22.255025Z","shell.execute_reply":"2021-11-15T21:36:34.974358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import segmentation_models as sm\nsm.set_framework('tf.keras')\nsm.framework()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:34.977504Z","iopub.execute_input":"2021-11-15T21:36:34.977949Z","iopub.status.idle":"2021-11-15T21:36:35.350083Z","shell.execute_reply.started":"2021-11-15T21:36:34.977892Z","shell.execute_reply":"2021-11-15T21:36:35.348904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## بناء الموديل","metadata":{}},{"cell_type":"code","source":"from segmentation_models import Unet\nfrom segmentation_models.utils import set_trainable\n\n\nmodel = Unet('efficientnetb0',input_shape=(256, 256, 3), classes=3, activation='sigmoid',encoder_weights='imagenet')\n#inp = Input(shape=(512, 640, 1))\n#l1 = Conv2D(3, (1, 1))(inp) # map N channels data to 3 channels\n#out = base_model(l1)\n#model = Model(inp, out, name=base_model.name)\n\nmodel.compile(optimizer='adam', loss=bce_dice_loss,metrics=[dice_coef,iou_coef,'accuracy']) #bce_dice_loss binary_crossentropy\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:36:35.351926Z","iopub.execute_input":"2021-11-15T21:36:35.352469Z","iopub.status.idle":"2021-11-15T21:36:42.221025Z","shell.execute_reply.started":"2021-11-15T21:36:35.352424Z","shell.execute_reply":"2021-11-15T21:36:42.219980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#تدريب الموديل
","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import Callback, ModelCheckpoint\ncheckpoint = ModelCheckpoint(\n 'model.h5', \n monitor='val_loss', \n verbose=0, \n save_best_only=True, \n save_weights_only=False,\n mode='auto'\n)\n\n\nhistory = model.fit(\n train_generator,\n validation_data=val_generator,\n callbacks=[checkpoint],\n use_multiprocessing=False,\n workers=4,\n epochs=1\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:39:16.120068Z","iopub.execute_input":"2021-11-15T21:39:16.122494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction=model.predect(train_generator)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_df = pd.DataFrame(history.history)\nhist_df.to_csv('history.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-15T21:39:10.450552Z","iopub.status.idle":"2021-11-15T21:39:10.451090Z","shell.execute_reply.started":"2021-11-15T21:39:10.450774Z","shell.execute_reply":"2021-11-15T21:39:10.450801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PLOT TRAINING\nplt.figure(figsize=(15,5))\nplt.plot(range(history.epoch[-1]+1),history.history['val_iou_coef'],label='Val_iou_coef')\nplt.plot(range(history.epoch[-1]+1),history.history['iou_coef'],label='Trn_iou_coef')\nplt.title('IOU'); plt.xlabel('Epoch'); plt.ylabel('iou_coef');plt.legend(); \nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PLOT TRAINING\nplt.figure(figsize=(15,5))\nplt.plot(range(history.epoch[-1]+1),history.history['val_dice_coef'],label='Val_dice_coef')\nplt.plot(range(history.epoch[-1]+1),history.history['dice_coef'],label='Trn_dice_coef')\nplt.title('DICE'); plt.xlabel('Epoch'); plt.ylabel('dice_coef');plt.legend(); \nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Referance:\n[here](https://www.kaggle.com/ammarnassanalhajali/sartorius-segmentation-keras-u-net-training)","metadata":{}}]}