{"cells":[{"metadata":{"_uuid":"f1aff4f1289eea4a91d2061871af57f0018733ce"},"cell_type":"markdown","source":"## Overview"},{"metadata":{"_uuid":"1d7c65b1b28b98b8d38dd994fb9dc9db91fc1cea"},"cell_type":"markdown","source":"It is a follow-up notebook to \"Fine-tuning ResNet34 on ship detection\" (https://www.kaggle.com/iafoss/fine-tuning-resnet34-on-ship-detection/notebook) and \"Unet34 (dice 0.87+)\" (https://www.kaggle.com/iafoss/unet34-dice-0-87/notebook) that shows how to evaluate the solution and submit predictions. Please check these notebooks for additional details."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from fastai.conv_learner import *\nfrom fastai.dataset import *\n\nimport pandas as pd\nimport numpy as np\nimport os\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tnrange, tqdm_notebook\nfrom scipy import ndimage","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76374df0da5adc6835cf988351c39ce3ce552641"},"cell_type":"code","source":"PATH = './'\nTRAIN = '../input/airbus-ship-detection/train_v2/'\nTEST = '../input/airbus-ship-detection/test_v2/'\nSEGMENTATION = '../input/airbus-ship-detection/train_ship_segmentations_v2.csv'\nPRETRAINED_DETECTION_PATH = '../input/fine-tuning-resnet34-on-ship-detection/models/'\nPRETRAINED_SEGMENTATION_PATH = '../input/unet34-dice-0-87/models/'\nDETECTION_TEST_PRED = '../input/fine-tuning-resnet34-on-ship-detection-new-data/ship_detection.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1b6a04600ea9e44401515d292067af1b5ab9cf8"},"cell_type":"code","source":"nw = 2   #number of workers for data loader\narch = resnet34 #specify target architecture","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7491c8ef4a4be7d4d437d950d814497de7b4268"},"cell_type":"markdown","source":"### Data"},{"metadata":{"trusted":true,"_uuid":"1d2bbf7abe0b64932d0fd9d7497cdd7d3853881a"},"cell_type":"code","source":"train_names = [f for f in os.listdir(TRAIN)]\ntest_names = [f for f in os.listdir(TEST)]\n#5% of data in the validation set is sufficient for model evaluation\ntr_n, val_n = train_test_split(train_names, test_size=0.05, random_state=42)\nsegmentation_df = pd.read_csv(os.path.join(PATH, SEGMENTATION)).set_index('ImageId')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0e06d20508ecbd80b929dde5b3cd1f0ea6e10167"},"cell_type":"markdown","source":"As explained in https://www.kaggle.com/iafoss/unet34-dice-0-87/notebook, I drop all images without ships. The model responsible for ship detection will take care of them."},{"metadata":{"trusted":true,"_uuid":"4f7c4937cf0cb83bc419977b198bcb71739e39db"},"cell_type":"code","source":"def cut_empty(names):\n    return [name for name in names \n            if(type(segmentation_df.loc[name]['EncodedPixels']) != float)]\n\ntr_n_cut = cut_empty(tr_n)\nval_n_cut = cut_empty(val_n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98426b544b2ec283847b05900cdef8fef96cca26"},"cell_type":"code","source":"def get_mask(img_id, df):\n    shape = (768,768)\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    masks = df.loc[img_id]['EncodedPixels']\n    if(type(masks) == float): return img.reshape(shape)\n    if(type(masks) == str): masks = [masks]\n    for mask in masks:\n        s = mask.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1\n    return img.reshape(shape).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4412b5292790c30d77f8320115765541e670cab5"},"cell_type":"code","source":"class pdFilesDataset(FilesDataset):\n    def __init__(self, fnames, path, transform):\n        self.segmentation_df = pd.read_csv(SEGMENTATION).set_index('ImageId')\n        super().__init__(fnames, transform, path)\n    \n    def get_x(self, i):\n        img = open_image(os.path.join(self.path, self.fnames[i]))\n        if self.sz == 768: return img \n        else: return cv2.resize(img, (self.sz, self.sz))\n    \n    def get_y(self, i):\n        mask = np.zeros((768,768), dtype=np.uint8) if (self.path == TEST) \\\n            else get_mask(self.fnames[i], self.segmentation_df)\n        img = Image.fromarray(mask).resize((self.sz, self.sz)).convert('RGB')\n        return np.array(img).astype(np.float32)\n    \n    def get_c(self): return 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0b680f127a97227a96d0b890fb3f26c474ff0f5"},"cell_type":"code","source":"def get_data(sz,bs):\n    tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)\n    tr_names = tr_n if (len(tr_n_cut)%bs == 0) else tr_n[:-(len(tr_n_cut)%bs)] #cut incomplete batch\n    ds = ImageData.get_ds(pdFilesDataset, (tr_names,TRAIN), \n                (val_n_cut,TRAIN), tfms, test=(test_names,TEST))\n    md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)\n    return md","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2c36d31bbf1a3afc3996222d450608f66bbb3d6"},"cell_type":"markdown","source":"### Model"},{"metadata":{"trusted":true,"_uuid":"785eb13d12159b2a90835e248093f2e4bd3d661a"},"cell_type":"code","source":"cut,lr_cut = model_meta[arch]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e9c3d8cdad5eaccbc2543a6f4c7c7b498c03888"},"cell_type":"code","source":"def get_base(pre=True):              #load ResNet34 model\n    layers = cut_model(arch(pre), cut)\n    return nn.Sequential(*layers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8147f86421b467087ec60dbddd0f1b88e88844cd"},"cell_type":"code","source":"class UnetBlock(nn.Module):\n    def __init__(self, up_in, x_in, n_out):\n        super().__init__()\n        up_out = x_out = n_out//2\n        self.x_conv  = nn.Conv2d(x_in,  x_out,  1)\n        self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)\n        self.bn = nn.BatchNorm2d(n_out)\n        \n    def forward(self, up_p, x_p):\n        up_p = self.tr_conv(up_p)\n        x_p = self.x_conv(x_p)\n        cat_p = torch.cat([up_p,x_p], dim=1)\n        return self.bn(F.relu(cat_p))\n\nclass SaveFeatures():\n    features=None\n    def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)\n    def hook_fn(self, module, input, output): self.features = output\n    def remove(self): self.hook.remove()\n    \nclass Unet34(nn.Module):\n    def __init__(self, rn):\n        super().__init__()\n        self.rn = rn\n        self.sfs = [SaveFeatures(rn[i]) for i in [2,4,5,6]]\n        self.up1 = UnetBlock(512,256,256)\n        self.up2 = UnetBlock(256,128,256)\n        self.up3 = UnetBlock(256,64,256)\n        self.up4 = UnetBlock(256,64,256)\n        self.up5 = nn.ConvTranspose2d(256, 1, 2, stride=2)\n        \n    def forward(self,x):\n        x = F.relu(self.rn(x))\n        x = self.up1(x, self.sfs[3].features)\n        x = self.up2(x, self.sfs[2].features)\n        x = self.up3(x, self.sfs[1].features)\n        x = self.up4(x, self.sfs[0].features)\n        x = self.up5(x)\n        return x[:,0]\n    \n    def close(self):\n        for sf in self.sfs: sf.remove()\n            \nclass UnetModel():\n    def __init__(self,model,name='Unet'):\n        self.model,self.name = model,name\n\n    def get_layer_groups(self, precompute):\n        lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))\n        return lgs + [children(self.model)[1:]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8de38bae09beab2d2acd7c824373cbd4b111309b"},"cell_type":"markdown","source":"### Score evaluation"},{"metadata":{"trusted":true,"_uuid":"8778da2e4fd46ccca6a551cc84a022f4c3c18140"},"cell_type":"code","source":"def IoU(pred, targs):\n    pred = (pred > 0.5).astype(float)\n    intersection = (pred*targs).sum()\n    return intersection / ((pred+targs).sum() - intersection + 1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"deee9be6cad6c5dde352e1022837e50550fa5a3c"},"cell_type":"code","source":"def get_score(pred, true):\n    n_th = 10\n    b = 4\n    thresholds = [0.5 + 0.05*i for i in range(n_th)]\n    n_masks = len(true)\n    n_pred = len(pred)\n    ious = []\n    score = 0\n    for mask in true:\n        buf = []\n        for p in pred: buf.append(IoU(p,mask))\n        ious.append(buf)\n    for t in thresholds:   \n        tp, fp, fn = 0, 0, 0\n        for i in range(n_masks):\n            match = False\n            for j in range(n_pred):\n                if ious[i][j] > t: match = True\n            if not match: fn += 1\n        \n        for j in range(n_pred):\n            match = False\n            for i in range(n_masks):\n                if ious[i][j] > t: match = True\n            if match: tp += 1\n            else: fp += 1\n        score += ((b+1)*tp)/((b+1)*tp + b*fn + fp)       \n    return score/n_th","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7e7ea344cecf23394740a013142b26320092b5c6"},"cell_type":"markdown","source":"In this competition we should submit and individual mask for each identified ship. The simplest way to do it is splitting the total mask into individual ones based on the connectivity of detected objects."},{"metadata":{"trusted":true,"_uuid":"23628b68164333efe7bb09b9690bc0ccd4138984"},"cell_type":"code","source":"def split_mask(mask):\n    threshold = 0.5\n    threshold_obj = 30 #ignor predictions composed of \"threshold_obj\" pixels or less\n    labled,n_objs = ndimage.label(mask > threshold)\n    result = []\n    for i in range(n_objs):\n        obj = (labled == i + 1).astype(int)\n        if(obj.sum() > threshold_obj): result.append(obj)\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0b03496a63d7115b005aad9b96ee40420662c61"},"cell_type":"code","source":"def get_mask_ind(img_id, df, shape = (768,768)): #return mask for each ship\n    masks = df.loc[img_id]['EncodedPixels']\n    if(type(masks) == float): return []\n    if(type(masks) == str): masks = [masks]\n    result = []\n    for mask in masks:\n        img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n        s = mask.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1\n        result.append(img.reshape(shape).T)\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1c265c22c8c56ae3b4b00dbd7fc6f4761d66158"},"cell_type":"code","source":"class Score_eval():\n    def __init__(self):\n        self.segmentation_df = pd.read_csv(SEGMENTATION).set_index('ImageId')\n        self.score, self.count = 0.0, 0\n        \n    def put(self,pred,name):\n        true = get_mask_ind(name, self.segmentation_df)\n        self.score += get_score(pred,true)\n        self.count += 1\n        \n    def evaluate(self):\n        return self.score/self.count","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3601c841b2b55b22286f7f6ac8304d38a1e1c11b"},"cell_type":"markdown","source":"### TTA\nDefine transformations for data augmentation and TTA function (default fast.ai functions do not transform a predicted mask back):"},{"metadata":{"trusted":true,"_uuid":"46320215f21f5682ebece222bbed6ce28ee4784e"},"cell_type":"code","source":"def aug_unit(x,fwd=True,mask=False):\n    return x\n\ndef aug_flipV(x,fwd=True,mask=False):\n    return x.flip(2) if mask else x.flip(3)\n\ndef aug_flipH(x,fwd=True,mask=False):\n    return x.flip(1) if mask else x.flip(2)\n\ndef aug_T(x,fwd=True,mask=False):\n    return torch.transpose(x,1,2) if mask else torch.transpose(x,2,3)\n\ndef aug_rot_2(x,fwd=True,mask=False): #rotate pi/2\n    return aug_flipV(aug_flipH(x,fwd,mask),fwd,mask)\n\ndef aug_rot_4cr(x,fwd=True,mask=False): #rotate pi/4 counterclockwise\n    return aug_flipV(aug_T(x,fwd,mask),fwd,mask) if fwd else \\\n        aug_T(aug_flipV(x,fwd,mask),fwd,mask)\n\ndef aug_rot_4cw(x,fwd=True,mask=False): #rotate pi/4 clockwise\n    return aug_flipH(aug_T(x,fwd,mask),fwd,mask) if fwd else \\\n        aug_T(aug_flipH(x,fwd,mask),fwd,mask)\n\ndef aug_rot_2T(x,fwd=True,mask=False): #transpose and rotate pi/2\n    return aug_rot_2(aug_T(x,fwd,mask),fwd,mask)\n\ntrms_side_on = [aug_unit,aug_flipH]\ntrms_top_down = [aug_unit,aug_flipV]\ntrms_dihedral = [aug_unit,aug_flipH,aug_flipV,aug_T,aug_rot_2,aug_rot_2T,\n                 aug_rot_4cw,aug_rot_4cr]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82463057f489b7f6cf415233558edbaa58c801c4"},"cell_type":"code","source":"def enc_img(img):\n    return torch.transpose(torch.tensor(img),0,2).unsqueeze(0)\n\ndef dec_img(img):\n    return to_np(torch.transpose(img.squeeze(0),0,2))\n\ndef display_augs(x,augs=aug_unit):\n    columns = 4\n    n = len(augs)\n    rows = n//4 + 1\n    fig=plt.figure(figsize=(columns*4, rows*4))\n    img = enc_img(x)\n    for i in range(rows):\n        for j in range(columns):\n            idx = j+i*columns\n            if idx >= n: break\n            fig.add_subplot(rows, columns, idx+1)\n            plt.axis('off')\n            plt.imshow(dec_img(augs[idx](img)))\n    plt.show()\n    \nimg = np.array(Image.open(os.path.join(TRAIN,'ce69faa4b.jpg')))\ndisplay_augs(img,trms_dihedral)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f715b343f2accfb8b0b9bbdc9fd6638c6d893da9"},"cell_type":"markdown","source":"Since the model predicts pixel masks, which are quite large, running standard functions for making a prediction will fail due to memory issue, especially for the test set, where about 100k 786x786 pixel masks should be created. Therefore, I wrote a function that does prediction batch by batch and applies F_save function for each generated mask."},{"metadata":{"trusted":true,"_uuid":"fa089fd879fb07d47ef774785a1e0a2b00046d38"},"cell_type":"code","source":"def model_pred(learner, dl, F_save): #if use train dl, disable shuffling\n    learner.model.eval();\n    name_list = dl.dataset.fnames\n    num_batchs = len(dl)\n    t = tqdm(iter(dl), leave=False, total=num_batchs)\n    count = 0\n    for x,y in t:\n        py = to_np(torch.sigmoid(learn.model(V(x))))\n        batch_size = len(py)\n        for i in range(batch_size):\n            F_save(py[i],to_np(y[i]),name_list[count])\n            count += 1\n            \ndef pred_aug(x,aug=[aug_unit]):\n    pred = []\n    for aug_cur in aug:\n        py = to_np(aug_cur(torch.sigmoid(learn.model(V(aug_cur(x)))),\n                           fwd=False, mask=True))\n        pred.append(py)\n    pred = np.stack(pred, axis=0).mean(axis=0)\n    return pred\n\n#if use train dl, disable shuffling\ndef model_pred_aug(learner, dl, F_save, aug=[aug_unit]):\n    learner.model.eval();\n    name_list = dl.dataset.fnames\n    num_batchs = len(dl)\n    t = tqdm(iter(dl), leave=False, total=num_batchs)\n    count = 0\n    for x,y in t:\n        pred = pred_aug(x,aug)           \n        batch_size = len(pred)\n        for i in range(batch_size):\n            F_save(pred[i],to_np(y[i]),name_list[count])\n            count += 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fa7634b20774b7c2293d5f15f6f37ab2e19f85a2"},"cell_type":"markdown","source":"### Prediction"},{"metadata":{"trusted":true,"_uuid":"c9700cc48f14aed6ef66024dccb5da51ab5df004"},"cell_type":"code","source":"m = to_gpu(Unet34(get_base(False)))\nmodels = UnetModel(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"741a0f34d5da2ac0a97f88a32685a10d5771950e"},"cell_type":"code","source":"sz = 768 #image size\nbs = 8  #batch size\nmd = get_data(sz,bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cb66dcc8fb4a8ee7d583277199e6ca52b9a094a"},"cell_type":"code","source":"learn = ConvLearner(md, models)\nlearn.models_path = PRETRAINED_SEGMENTATION_PATH\nlearn.load('Unet34_768_1')\nlearn.models_path = PATH","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"509e3579f27dfd404b575f8475a35a50759aef41"},"cell_type":"markdown","source":"Running the model evaluation on the validation set."},{"metadata":{"trusted":true,"_uuid":"b1aa3ec5340bbb698142cbb89755ba5e631d6bb1","scrolled":true},"cell_type":"code","source":"score = Score_eval()\nprocess_pred = lambda yp, y, name : score.put(split_mask(yp),name)\nmodel_pred_aug(learn, md.val_dl, process_pred, trms_dihedral)\nprint('\\n',score.evaluate())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b2b0ed553655a84761ac02bf3894a8d6703d344"},"cell_type":"markdown","source":"It is the **score based only on images with ships**, a model responsible for ship detection (accuracy ~98%) takes care of images without ships. Since the fraction of empty images in the test set is 0.52, the expected score of the model stacked with ship detection one (https://www.kaggle.com/iafoss/fine-tuning-resnet34-on-ship-detection/notebook) is approximately 0.52 + 0.37 x 0.48 = 0.70 (if the new test set is similar to old training data). However, you should keep in mind that the evaluated model has been trained only for one epoch on full resolution images (the dice is only ~0.80 for 784x784 images). Continuing training the model and mask postprocessing can further boost it."},{"metadata":{"_uuid":"a4308d215ee4970ebf0ce80964b826c9fa4bf630"},"cell_type":"markdown","source":"### Submission\nLoad the prediction of ship detection model (https://www.kaggle.com/iafoss/fine-tuning-resnet34-on-ship-detection/notebook) for the test set."},{"metadata":{"trusted":true,"_uuid":"4060cbb4a43de4899dbab4ed2f4cf3d557431ed9"},"cell_type":"code","source":"ship_detection = pd.read_csv(DETECTION_TEST_PRED)\nship_detection.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"95bb6b7a994acc27454d192d1bc7bb9de8a41876"},"cell_type":"markdown","source":"Identify images with ships and run Unet34 model only for them.  It looks that there is some descripancy here since the fraction of empty images in public LB acording to empty submission test is 0.52."},{"metadata":{"trusted":true,"_uuid":"f0923fdae18aceffcdbf6b067e2e78bf794e1cb8"},"cell_type":"code","source":"test_names = ship_detection.loc[ship_detection['p_ship'] > 0.5, ['id']]['id'].values.tolist()\ntest_names_nothing = ship_detection.loc[ship_detection['p_ship'] <= 0.5, ['id']]['id'].values.tolist()\nlen(test_names), len(test_names_nothing)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8b37e28cc5dc23d6769aa0765eaeadb4df9a0de"},"cell_type":"code","source":"md = get_data(sz,bs)\nlearn.set_data(md)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8bb0a20538d9e51a3c1efbeaecbe064d7653f863"},"cell_type":"markdown","source":"The function for mask decoding is borrowed from https://www.kaggle.com/kmader/from-trained-u-net-to-submission-part-2/notebook ."},{"metadata":{"trusted":true,"_uuid":"ec3a58469656ee1b7040d37ba36f6f1344753767"},"cell_type":"code","source":"def decode_mask(mask, shape=(768, 768)):\n    pixels = mask.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0586084f54c369e9e8f5a6cbe4855983f7404c50"},"cell_type":"markdown","source":"Predict masks only for images that are Identified to have ships."},{"metadata":{"trusted":true,"_uuid":"227ebf463005e664a9e8c0728b7aeb2c69765c60"},"cell_type":"code","source":"ship_list_dict = []\nfor name in test_names_nothing:\n    ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e9f634e2e81c766c27179234fd843a73b35bf0c"},"cell_type":"code","source":"def enc_test(yp, y, name):\n    masks = split_mask(yp)\n    if(len(masks) == 0): \n        ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})\n    for mask in masks:\n        ship_list_dict.append({'ImageId':name,'EncodedPixels':decode_mask(mask)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e311f443432e40efdfb6b6a55c3891689f6aa2b3"},"cell_type":"code","source":"model_pred_aug(learn, md.test_dl, enc_test, trms_dihedral)\npred_df = pd.DataFrame(ship_list_dict)\npred_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}