{"cells":[{"metadata":{"_uuid":"53a09f47680e7fcfa769e563d618f7d6f4772faf"},"cell_type":"markdown","source":"**\nAlough I failed on this competition I want to share this ensemble model\nI would make the comeback next segmentation competition haha/font>**"},{"metadata":{"trusted":true,"_uuid":"96c977ca23da47eae57fce19f6252ac257272a37","scrolled":true},"cell_type":"code","source":"from fastai.conv_learner import *\nfrom fastai.dataset import *\n\nimport pandas as pd\nimport numpy as np\nimport os\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tnrange, tqdm_notebook\nfrom scipy import ndimage\nfrom keras.backend import tf as ktf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70e6e1883181fb2a007f4fc25c543b9b16d06007"},"cell_type":"code","source":"print(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f63c0f897a608c9309b7300e63bb9e79c36e737c"},"cell_type":"code","source":"PATH = './'\nTRAIN = '../input/airbus-ship-detection/train_v2/'\nTEST = '../input/airbus-ship-detection/test_v2/'\nSEGMENTATION = '../input/airbus-ship-detection/train_ship_segmentations_v2.csv'\nPRETRAINED_DETECTION_PATH = '../input/fine-tuning-resnet34-on-ship-detection/models/'\nPRETRAINED_SEGMENTATION_PATH = '../input/unet34-dice-0-87/models/'\nDETECTION_TEST_PRED = '../input/fine-tuning-resnet34-ship-redo/Have_ship_or_not.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"876a6834e33e0a5c9a1b6f13d1de982f85ff2e70"},"cell_type":"code","source":"nw = 2 #number of workers for data loader\narch = resnet34 #specify target architecture","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15894135568da2912e6c49990f661accd2912dbe"},"cell_type":"code","source":"train_names = [f for f in os.listdir(TRAIN)][:1000]\ntest_names = [f for f in os.listdir(TEST)][:1000]\n#5% of data in the validation set is sufficient for model evaluation\ntr_n, val_n = train_test_split(train_names, test_size=0.05, random_state=42)\nsegmentation_df = pd.read_csv(os.path.join(PATH, SEGMENTATION)).set_index('ImageId')\n\ndef cut_empty(names):\n return [name for name in names \n if(type(segmentation_df.loc[name]['EncodedPixels']) != float)]\n\ntr_n_cut = cut_empty(tr_n)\nval_n_cut = cut_empty(val_n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6878f96efd90436e8fd6ed5e665636427183ef4c"},"cell_type":"code","source":"def get_mask(img_id, df):\n shape = (768,768)\n img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n masks = df.loc[img_id]['EncodedPixels']\n if(type(masks) == float): return img.reshape(shape)\n if(type(masks) == str): masks = [masks]\n for mask in masks:\n s = mask.split()\n for i in range(len(s)//2):\n start = int(s[2*i]) - 1\n length = int(s[2*i+1])\n img[start:start+length] = 1\n return img.reshape(shape).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d56b9e3414a2ec1669d33d3fff78eb1a4f57f673"},"cell_type":"code","source":"class pdFilesDataset(FilesDataset):\n def __init__(self, fnames, path, transform):\n self.segmentation_df = pd.read_csv(SEGMENTATION).set_index('ImageId')\n super().__init__(fnames, transform, path)\n \n def get_x(self, i):\n img = open_image(os.path.join(self.path, self.fnames[i]))\n if self.sz == 768: return img \n else: return cv2.resize(img, (self.sz, self.sz))\n \n def get_y(self, i):\n mask = np.zeros((768,768), dtype=np.uint8) if (self.path == TEST) \\\n else get_mask(self.fnames[i], self.segmentation_df)\n img = Image.fromarray(mask).resize((self.sz, self.sz)).convert('RGB')\n return np.array(img).astype(np.float32)\n \n def get_c(self): return 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"253029be70ddfde9b4ca6e28af486f516b146688"},"cell_type":"code","source":"def get_data(sz,bs):\n tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)\n tr_names = tr_n if (len(tr_n_cut)%bs == 0) else tr_n[:-(len(tr_n_cut)%bs)] #cut incomplete batch\n ds = ImageData.get_ds(pdFilesDataset, (tr_names,TRAIN), \n (val_n_cut,TRAIN), tfms, test=(test_names,TEST))\n md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)\n return md","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f291bee3317c81754ebb595d9b06f0433d76e448"},"cell_type":"code","source":"cut,lr_cut = model_meta[arch]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22d682cbb82c64cb9cf28d7e62ea14b13fd50d4d"},"cell_type":"code","source":"def get_base(pre=True): #load ResNet34 model\n layers = cut_model(arch(pre), cut)\n return nn.Sequential(*layers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efc26934e6f14369cc0dcf0f0053eb485b703e20"},"cell_type":"code","source":"class UnetBlock(nn.Module):\n def __init__(self, up_in, x_in, n_out):\n super().__init__()\n up_out = x_out = n_out//2\n self.x_conv = nn.Conv2d(x_in, x_out, 1)\n self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)\n self.bn = nn.BatchNorm2d(n_out)\n \n def forward(self, up_p, x_p):\n up_p = self.tr_conv(up_p)\n x_p = self.x_conv(x_p)\n cat_p = torch.cat([up_p,x_p], dim=1)\n return self.bn(F.relu(cat_p))\n\nclass SaveFeatures():\n features=None\n def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)\n def hook_fn(self, module, input, output): self.features = output\n def remove(self): self.hook.remove()\n \nclass Unet34(nn.Module):\n def __init__(self, rn):\n super().__init__()\n self.rn = rn\n self.sfs = [SaveFeatures(rn[i]) for i in [2,4,5,6]]\n self.up1 = UnetBlock(512,256,256)\n self.up2 = UnetBlock(256,128,256)\n self.up3 = UnetBlock(256,64,256)\n self.up4 = UnetBlock(256,64,256)\n self.up5 = nn.ConvTranspose2d(256, 1, 2, stride=2)\n \n def forward(self,x):\n x = F.relu(self.rn(x))\n x = self.up1(x, self.sfs[3].features)\n x = self.up2(x, self.sfs[2].features)\n x = self.up3(x, self.sfs[1].features)\n x = self.up4(x, self.sfs[0].features)\n x = self.up5(x)\n return x[:,0]\n \n def close(self):\n for sf in self.sfs: sf.remove()\n \nclass UnetModel():\n def __init__(self,model,name='Unet'):\n self.model,self.name = model,name\n\n def get_layer_groups(self, precompute):\n lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))\n return lgs + [children(self.model)[1:]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62a29d956e77e139ecdb1610eadcb4d6dca6c66c"},"cell_type":"code","source":"def split_mask(mask):\n threshold = 0.57777778\n threshold_obj = 30 #ignor predictions composed of \"threshold_obj\" pixels or less\n labled,n_objs = ndimage.label(mask > threshold)\n result = []\n for i in range(n_objs):\n obj = (labled == i + 1).astype(int)\n if(obj.sum() > threshold_obj): result.append(obj)\n return result\ndef get_mask_ind(img_id, df, shape = (768,768)): #return mask for each ship\n masks = df.loc[img_id]['EncodedPixels']\n if(type(masks) == float): return []\n if(type(masks) == str): masks = [masks]\n result = []\n for mask in masks:\n img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n s = mask.split()\n for i in range(len(s)//2):\n start = int(s[2*i]) - 1\n length = int(s[2*i+1])\n img[start:start+length] = 1\n result.append(img.reshape(shape).T)\n return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38a229fdd0253d831d07285b4abeb128083ced96"},"cell_type":"code","source":"def aug_unit(x,fwd=True,mask=False):\n return x\n\ndef aug_flipV(x,fwd=True,mask=False):\n return x.flip(2) if mask else x.flip(3)\n\ndef aug_flipH(x,fwd=True,mask=False):\n return x.flip(1) if mask else x.flip(2)\n\ndef aug_T(x,fwd=True,mask=False):\n return torch.transpose(x,1,2) if mask else torch.transpose(x,2,3)\n\ndef aug_rot_2(x,fwd=True,mask=False): #rotate pi/2\n return aug_flipV(aug_flipH(x,fwd,mask),fwd,mask)\n\ndef aug_rot_4cr(x,fwd=True,mask=False): #rotate pi/4 counterclockwise\n return aug_flipV(aug_T(x,fwd,mask),fwd,mask) if fwd else \\\n aug_T(aug_flipV(x,fwd,mask),fwd,mask)\n\ndef aug_rot_4cw(x,fwd=True,mask=False): #rotate pi/4 clockwise\n return aug_flipH(aug_T(x,fwd,mask),fwd,mask) if fwd else \\\n aug_T(aug_flipH(x,fwd,mask),fwd,mask)\n\ndef aug_rot_2T(x,fwd=True,mask=False): #transpose and rotate pi/2\n return aug_rot_2(aug_T(x,fwd,mask),fwd,mask)\n\ntrms_side_on = [aug_unit,aug_flipH]\ntrms_top_down = [aug_unit,aug_flipV]\ntrms_dihedral = [aug_unit,aug_flipH,aug_flipV,aug_T,aug_rot_2,aug_rot_2T,\n aug_rot_4cw,aug_rot_4cr]\ndef enc_img(img):\n return torch.transpose(torch.tensor(img),0,2).unsqueeze(0)\n\ndef dec_img(img):\n return to_np(torch.transpose(img.squeeze(0),0,2))\n\ndef display_augs(x,augs=aug_unit):\n columns = 4\n n = len(augs)\n rows = n//4 + 1\n fig=plt.figure(figsize=(columns*4, rows*4))\n img = enc_img(x)\n for i in range(rows):\n for j in range(columns):\n idx = j+i*columns\n if idx >= n: break\n fig.add_subplot(rows, columns, idx+1)\n plt.axis('off')\n plt.imshow(dec_img(augs[idx](img)))\n plt.show()\n \nimg = np.array(Image.open(os.path.join(TRAIN,'ce69faa4b.jpg')))\ndisplay_augs(img,trms_dihedral)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b43eb666144a32af8631246868a1b9f3be5de0c3"},"cell_type":"code","source":"m = to_gpu(Unet34(get_base(False)))\nmodels = UnetModel(m)\nsz = 768 #image size\nbs = 1 #batch size\nmd = get_data(sz,bs)\nlearn = ConvLearner(md, models)\nlearn.models_path = PRETRAINED_SEGMENTATION_PATH\nlearn.load('Unet34_768_1')\nlearn.models_path = PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"595b2afd58bf7ceeb663094fc4361389f1ee28a3"},"cell_type":"code","source":"#########################keras model###############################","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"535b030be59c7a3860526ba872f051c3c7e95387"},"cell_type":"code","source":"import tensorflow as ktf\nfrom keras.backend.tensorflow_backend import set_session\nconfig = ktf.ConfigProto()\nconfig.gpu_options.per_process_gpu_memory_fraction = 0.4\nconfig.gpu_options.visible_device_list = \"0\"\nset_session(ktf.Session(config=config))\nprint('ktf limited')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55bdeebfe47b255f75efc9f73f9eae8cb7304359"},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom skimage.io import imread\nimport matplotlib.pyplot as plt\nfrom skimage.segmentation import mark_boundaries\nfrom skimage.util.montage import montage2d as montage\nmontage_rgb = lambda x: np.stack([montage(x[:, :, :, i]) for i in range(x.shape[3])], -1)\nship_dir = '../input/airbus-ship-detection/'\ntrain_image_dir = os.path.join(ship_dir, 'train_v2')\ntest_image_dir = os.path.join(ship_dir, 'test_v2')\nexclude_list = ['6384c3e78.jpg','13703f040.jpg', '14715c06d.jpg', '33e0ff2d5.jpg',\n '4d4e09f2a.jpg', '877691df8.jpg', '8b909bb20.jpg', 'a8d99130e.jpg', \n 'ad55c3143.jpg', 'c8260c541.jpg', 'd6c7f17c7.jpg', 'dc3e7c901.jpg',\n 'e44dffe88.jpg', 'ef87bad36.jpg', 'f083256d8.jpg']\n\nimport gc; gc.enable() # memory is tight\nfrom skimage.morphology import label\n\ndef multi_rle_encode(img, **kwargs):\n '''\n Encode connected regions as separated masks\n '''\n labels = label(img)\n if img.ndim > 2:\n return [rle_encode(np.sum(labels==k, axis=2), **kwargs) for k in np.unique(labels[labels>0])]\n else:\n return [rle_encode(labels==k, **kwargs) for k in np.unique(labels[labels>0])]\n\n# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_encode(img):\n '''\n img: numpy array, 1 - mask, 0 - background\n Returns run length as string formated\n '''\n pixels = img.T.flatten()\n pixels = np.concatenate([[0], pixels, [0]])\n runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n runs[1::2] -= runs[::2]\n return ' '.join(str(x) for x in runs)\n\ndef rle_decode(mask_rle, shape=(768, 768)):\n '''\n mask_rle: run-length as string formated (start length)\n shape: (height,width) of array to return \n Returns numpy array, 1 - mask, 0 - background\n '''\n s = mask_rle.split()\n starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n starts -= 1\n ends = starts + lengths\n img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n for lo, hi in zip(starts, ends):\n img[lo:hi] = 1\n return img.reshape(shape).T # Needed to align to RLE direction\n\ndef masks_as_image(in_mask_list):\n # Take the individual ship masks and create a single mask array for all ships\n all_masks = np.zeros((768, 768), dtype = np.int16)\n #if isinstance(in_mask_list, list):\n for mask in in_mask_list:\n if isinstance(mask, str):\n all_masks += rle_decode(mask)\n return np.expand_dims(all_masks, -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62923d08dfef9d8b6c3f0242660fde266b7d9d62"},"cell_type":"code","source":"masks = pd.read_csv(os.path.join('../input/airbus-ship-detection/',\n 'train_ship_segmentations_v2.csv')).head(5000)\n\nmasks['corrupted'] = masks['ImageId'].map(lambda x: 1 if x in exclude_list else 0)\nprint(masks['corrupted'].value_counts())\nmasks = masks[masks['corrupted'] == 0]\nmasks = masks.drop(['corrupted'], axis=1)\n\nprint(masks.shape[0], 'masks found')\nprint(masks['ImageId'].value_counts().shape[0])\nmasks.head()\n\nmasks['ships'] = masks['EncodedPixels'].map(lambda c_row: 1 if isinstance(c_row, str) else 0)\nunique_img_ids = masks.groupby('ImageId').agg({'ships': 'sum'}).reset_index()\nunique_img_ids = unique_img_ids[unique_img_ids['ships']>3]\nunique_img_ids['has_ship'] = unique_img_ids['ships'].map(lambda x: 1.0 if x>0 else 0.0)\nunique_img_ids['has_ship_vec'] = unique_img_ids['has_ship'].map(lambda x: [x])\n# some files are too small/corrupt\nunique_img_ids['file_size_kb'] = unique_img_ids['ImageId'].map(lambda c_img_id: \n os.stat(os.path.join(train_image_dir, \n c_img_id)).st_size/1024)\nunique_img_ids = unique_img_ids[unique_img_ids['file_size_kb'] > 50] # keep only +50kb files\nunique_img_ids['file_size_kb'].hist()\nmasks.drop(['ships'], axis=1, inplace=True)\nunique_img_ids.sample(7)\n\nfrom sklearn.model_selection import train_test_split\ntrain_ids, valid_ids = train_test_split(unique_img_ids, \n test_size = 0.3, \n stratify = unique_img_ids['ships'])\ntrain_df = pd.merge(masks, train_ids)\nvalid_df = pd.merge(masks, valid_ids)\nprint(train_df.shape[0], 'training masks')\nprint(valid_df.shape[0], 'validation masks')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d7f05c30d098d7aed322fcc1b17c7a96e794eb4"},"cell_type":"code","source":"train_df['grouped_ship_count'] = train_df['ships'].map(lambda x: (x+1)//2).clip(0, 7)\ndef sample_ships(in_df, base_rep_val=1500):\n if in_df['ships'].values[0]==0:\n return in_df.sample(base_rep_val//3) # even more strongly undersample no ships\n else:\n return in_df.sample(base_rep_val, replace=(in_df.shape[0]0.57777778, np.expand_dims(disk(2), -1))\n\ndef predict(img, path=test_image_dir):\n cur_seg, c_img = raw_prediction(img, path=path)\n return smooth(cur_seg), c_img\n\n## Get a sample of each group of ship count\n\n# [c_ax.axis('off') for c_ax in m_axs.flatten()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51ae2df149b776f4ec662759cc5e4d640f5c361c"},"cell_type":"code","source":"temptest = []\ndef get_data(sz,bs = 1):\n tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)\n tr_names = tr_n if (len(tr_n_cut)%bs == 0) else tr_n[:-(len(tr_n_cut)%bs)] #cut incomplete batch\n ds = ImageData.get_ds(pdFilesDataset, (tr_names,TRAIN), \n (val_n_cut,TRAIN), tfms ,test=(temptest,TRAIN))\n md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)\n return md\n\ndef split_mask(mask):\n threshold = 0.57777778\n threshold_obj = 30 #ignor predictions composed of \"threshold_obj\" pixels or less\n labled,n_objs = ndimage.label(mask > threshold)\n result = []\n for i in range(n_objs):\n obj = (labled == i + 1).astype(int)\n if(obj.sum() > threshold_obj): result.append(rle_encode(obj))\n return result\n\ndef decode_mask(mask, shape=(768, 768)):\n pixels = mask.T.flatten()\n pixels = np.concatenate([[0], pixels, [0]])\n runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n runs[1::2] -= runs[::2]\n return ' '.join(str(x) for x in runs)\n\ndef pred_aug(x,aug=[aug_unit]):\n pred = []\n for aug_cur in aug:\n py = to_np(aug_cur(torch.sigmoid(learn.model(V(aug_cur(x)))),\n fwd=False, mask=True))\n pred.append(py)\n pred = np.stack(pred, axis=0).mean(axis=0)\n return pred\n\n#if use train dl, disable shuffling\ndef model_pred_aug(learner, dl, aug=[aug_unit]):\n learner.model.eval();\n name_list = dl.dataset.fnames\n num_batchs = len(dl)\n t = tqdm(iter(dl), leave=False, total=num_batchs)\n count = 0\n for x,y in t:\n pred = pred_aug(x,aug) \n batch_size = len(pred)\n if len(pred) > 1:\n print('len(pred error)',len(pred))\n return pred[0]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8acdaea8f745331871218f5cd46fcd06f985aad4"},"cell_type":"code","source":"from keras import models, layers\nfrom keras.engine.topology import Input\nfrom keras.engine.training import Model\nfrom keras.layers.convolutional import Conv2D, UpSampling2D, Conv2DTranspose\nfrom keras.layers.core import Activation, SpatialDropout2D\nfrom keras.layers.merge import concatenate,add\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.core import Lambda\nfrom keras.backend import tf as ktf\nfullres_model = models.load_model('../input/attention-train3/seg_model.h5', compile=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ed8d545f5a6e2dcc07a3c7875f8c8b2d07c32a6"},"cell_type":"code","source":"## Get a sample of each group of ship count\nsamples = valid_df.groupby('ships').apply(lambda x: x.sample(1))\nfig, m_axs = plt.subplots(samples.shape[0], 5, figsize = (20, samples.shape[0]*4))\n\nfor (ax1, ax2, ax3, ax4, ax5), c_img_name in zip(m_axs, samples.ImageId.values):\n first_seg, first_img = raw_prediction(c_img_name, train_image_dir)\n ax1.imshow(first_img)\n ax1.set_title('Image: ' + c_img_name)\n reencoded = masks_as_color(multi_rle_encode(smooth(first_seg)[:, :, 0]))\n ax2.imshow(reencoded)\n ax2.set_title('Prediction1 Masks')\n \n temptest = [c_img_name]\n md = get_data(768,1)\n learn.set_data(md)\n first_seg2 = model_pred_aug(learn, md.test_dl, trms_dihedral)\n reencoded2 = masks_as_color(split_mask(first_seg2))\n ax3.imshow(reencoded2)\n ax3.set_title('Prediction2 lafoss Masks')\n \n ensemble = first_seg[:, :, 0]*0.5 + first_seg2*0.5\n reencoded3 = masks_as_color(split_mask(first_seg2))\n ax4.imshow(reencoded3)\n ax4.set_title('Ensemble Masks')\n \n ground_truth = masks_as_color(masks.query('ImageId==\"{}\"'.format(c_img_name))['EncodedPixels'])\n ax5.imshow(ground_truth)\n ax5.set_title('Ground Truth')\n \nfig.savefig('test_predictions.png')\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04e5bcbf2bce6a2998778dfc52f0b6a8bed1668c"},"cell_type":"code","source":"test_paths = os.listdir(test_image_dir)\nprint(len(test_paths), 'test images found')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1b19708566b06659f7ac3528ad5f63289a612be"},"cell_type":"code","source":"temptest = []\ndef get_data(sz,bs = 1):\n tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)\n tr_names = tr_n if (len(tr_n_cut)%bs == 0) else tr_n[:-(len(tr_n_cut)%bs)] #cut incomplete batch\n ds = ImageData.get_ds(pdFilesDataset, (tr_names,TRAIN), \n (val_n_cut,TRAIN), tfms ,test=(temptest,TEST))\n md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)\n return md","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7b571505aa20029411f85713e9af3e509de68ad"},"cell_type":"code","source":"from tqdm import tqdm_notebook\nfrom skimage.morphology import binary_opening, disk\n\nfig, m_axs = plt.subplots(5, 4, figsize = (20,60))\nfor (ax1, ax2, ax3, ax4), c_img_name in zip(m_axs, test_paths):\n c_path = os.path.join(test_image_dir, c_img_name)\n c_img = imread(c_path)\n first_img = np.expand_dims(c_img, 0)/255.0\n \n ax1.imshow(first_img[0])\n ax1.set_title('Image: ' + c_img_name)\n \n first_seg = fullres_model.predict(first_img)[0]\n first_seg_0_5 = binary_opening(first_seg>0.57777778, np.expand_dims(disk(2), -1))[:, :, 0]\n ax2.imshow(first_seg_0_5)\n ax2.set_title('Prediction1 Masks')\n \n \n temptest = [c_img_name]\n md = get_data(768,1)\n learn.set_data(md)\n first_seg2 = model_pred_aug(learn, md.test_dl, trms_dihedral)\n reencoded2 = masks_as_color(split_mask(first_seg2))\n ax3.imshow(reencoded2)\n ax3.set_title('Prediction2 lafoss Masks')\n \n ensemble = first_seg2*0.5 + first_seg[:, :, 0]*0.5\n reencoded3 = binary_opening(ensemble>0.57777778)\n ax4.imshow(reencoded3)\n ax4.set_title('Ensemble Masks')\n \nfig.savefig('test_predictions.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f25dca29c16047283f6ce5697a5d45abe93e1f64"},"cell_type":"code","source":"have_ship= pd.read_csv(\"../input/fine-tuning-resnet34-ship-redo/ship_detection.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd89797a3e4c673e84c2f7d41cb3469efb1695e0"},"cell_type":"code","source":"have_ship.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ec8224506aa1920071cca42927dc07f08344346"},"cell_type":"code","source":"test_names = have_ship.loc[have_ship['p_ship'] > 0.5, ['id']]['id'].values.tolist()\ntest_names_nothing = have_ship.loc[have_ship['p_ship'] <= 0.5, ['id']]['id'].values.tolist()\nlen(test_names), len(test_names_nothing)\n\ntest_paths = os.listdir(test_image_dir)\nprint(len(test_paths), 'test images found')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3beae7627800b12dc6e545c5414465f6c2383f26"},"cell_type":"code","source":"ship_list_dict_64 = []\nship_list_dict_73 = []\nship_list_dict_73_more = []\nship_list_dict_82 = []\nship_list_dict_82_more = []\n\nship_list_dict_en_64 = []\nfor name in test_names_nothing:\n ship_list_dict_64.append({'ImageId':name,'EncodedPixels':None})\n ship_list_dict_73.append({'ImageId':name,'EncodedPixels':None})\n ship_list_dict_73_more.append({'ImageId':name,'EncodedPixels':None})\n ship_list_dict_82.append({'ImageId':name,'EncodedPixels':None})\n ship_list_dict_82_more.append({'ImageId':name,'EncodedPixels':None})\nlen(ship_list_dict_64),len(ship_list_dict_73)\n# ,len(ship_list_dict_73_more),len(ship_list_dict_82),len(ship_list_dict_82_more)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d6aacaa2f098c7b5d9a9f05be3be37e32a34bb6"},"cell_type":"code","source":"from tqdm import tqdm_notebook\nfrom skimage.morphology import binary_opening, disk\nfor c_img_name in tqdm_notebook(test_paths):\n if c_img_name in test_names:\n c_path = os.path.join(test_image_dir, c_img_name)\n c_img = imread(c_path)\n first_img = np.expand_dims(c_img, 0)/255.0\n first_seg = fullres_model.predict(first_img)[0]\n\n\n temptest = [c_img_name]\n md = get_data(768,1)\n learn.set_data(md)\n first_seg2 = model_pred_aug(learn, md.test_dl, trms_dihedral)\n\n\n ensemble6_4 = first_seg2*0.6 + first_seg[:, :, 0]*0.4 \n ensemble7_3 = first_seg2*0.7 + first_seg[:, :, 0]*0.3\n ensemble7_3_more = ensemble7_3 * 1.05\n ensemble6_4_masks = split_mask(ensemble6_4)\n ensemble7_3_masks = split_mask(ensemble7_3)\n ensemble7_3_more_masks = split_mask(ensemble7_3_more)\n\n if(len(ensemble6_4_masks) == 0): \n ship_list_dict_64.append({'ImageId':c_img_name,'EncodedPixels':np.nan})\n for mask in ensemble6_4_masks:\n ship_list_dict_64.append({'ImageId':c_img_name,'EncodedPixels':mask})\n\n if(len(ensemble7_3_masks) == 0): \n ship_list_dict_73.append({'ImageId':c_img_name,'EncodedPixels':np.nan})\n for mask in ensemble7_3_masks:\n ship_list_dict_73.append({'ImageId':c_img_name,'EncodedPixels':mask})\n\n ensemble6_4 = None\n ensemble7_3 = None\n ensemble6_4_masks = None\n ensemble7_3_masks = None\n gc.collect()\n\n ensemble8_2 = first_seg2*0.8 + first_seg[:, :, 0]*0.2\n ensemble8_2_more = ensemble8_2*1.05\n ensemble8_2_masks = split_mask(ensemble8_2)\n ensemble8_2_more_masks = split_mask(ensemble8_2_more)\n\n if(len(ensemble8_2_masks) == 0): \n ship_list_dict_82.append({'ImageId':c_img_name,'EncodedPixels':np.nan})\n for mask in ensemble8_2_masks:\n ship_list_dict_82.append({'ImageId':c_img_name,'EncodedPixels':mask})\n\n if(len(ensemble8_2_more_masks) == 0): \n ship_list_dict_82_more.append({'ImageId':c_img_name,'EncodedPixels':np.nan})\n for mask in ensemble8_2_more_masks:\n ship_list_dict_82_more.append({'ImageId':c_img_name,'EncodedPixels':mask})\n\n ensemble8_2 = None\n ensemble8_2_more = None\n ensemble8_2_masks = None\n ensemble8_2_more_masks = None\n gc.collect()\n \nlen(ship_list_dict_64),len(ship_list_dict_73)\n# ,len(ship_list_dict_73_more)\n# ,len(ship_list_dict_82),len(ship_list_dict_82_more)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d824ddd4340b116653b3076913279715983ca70"},"cell_type":"code","source":"submission_64 = pd.DataFrame(ship_list_dict_64)[['ImageId', 'EncodedPixels']]\nsubmission_64.to_csv('submission_64.csv', index=False)\nsubmission_64.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3e7243c31251ed079478e2a9faab6ad283389cf"},"cell_type":"code","source":"submission_73 = pd.DataFrame(ship_list_dict_73)[['ImageId', 'EncodedPixels']]\nsubmission_73.to_csv('submission_73.csv', index=False)\nsubmission_73.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd927ec25e005da43f7c641c3ab61f9fbef8a958"},"cell_type":"code","source":"submission_73_more = pd.DataFrame(ship_list_dict_73_more)[['ImageId', 'EncodedPixels']]\nsubmission_73_more.to_csv('submission_73_more.csv', index=False)\nsubmission_73_more.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b98cace133257a9ee8dc641d34acdd516ccc836"},"cell_type":"code","source":"submission_82 = pd.DataFrame(ship_list_dict_82)[['ImageId', 'EncodedPixels']]\nsubmission_82.to_csv('submission_82.csv', index=False)\nsubmission_82.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c64d73bb91238572be8f5745241fc4b55ac8824"},"cell_type":"code","source":"submission_82_more = pd.DataFrame(ship_list_dict_82_more)[['ImageId', 'EncodedPixels']]\nsubmission_82_more.to_csv('submission_82_more.csv', index=False)\nsubmission_82_more.sample(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a982e6453a20090bc0bc3f8ab4b8e806b4b108cb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e009c003a10f4591e8c2aa1e4ab890ca6a4ee124"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b23d143114b0391843b03824a9706f550b6894a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2726181f996baf27e0282c0db3f170d2b2225ce"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}