{"cells":[{"metadata":{"_uuid":"859744fa6977a2428a448cb22f745e3f2613a428"},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"f1aff4f1289eea4a91d2061871af57f0018733ce"},"cell_type":"markdown","source":"## Overview"},{"metadata":{"_uuid":"1d7c65b1b28b98b8d38dd994fb9dc9db91fc1cea"},"cell_type":"markdown","source":"It is a follow-up notebook to \"Fine-tuning ResNet34 on ship detection\" (https://www.kaggle.com/iafoss/fine-tuning-resnet34-on-ship-detection/notebook) and \"Unet34 (dice 0.87+)\" (https://www.kaggle.com/iafoss/unet34-dice-0-87/notebook) that shows how to evaluate the solution and submit predictions. Please check these notebooks for additional details."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"from fastai.conv_learner import *\nfrom fastai.dataset import *\n\nimport pandas as pd\nimport numpy as np\nimport os\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tnrange, tqdm_notebook\nfrom scipy import ndimage","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76374df0da5adc6835cf988351c39ce3ce552641","collapsed":true},"cell_type":"code","source":"PATH = './'\nTRAIN = '../input/airbus-ship-detection/train/'\nTEST = '../input/airbus-ship-detection/test/'\nSEGMENTATION = '../input/airbus-ship-detection/train_ship_segmentations.csv'\nPRETRAINED_DETECTION_PATH = '../input/fine-tuning-resnet34-on-ship-detection/models/'\nPRETRAINED_SEGMENTATION_PATH = '../input/unet34-dice-0-87/models/'\nDETECTION_TEST_PRED = '../input/fine-tuning-resnet34-on-ship-detection/ship_detection.csv'\nexclude_list = ['6384c3e78.jpg','13703f040.jpg', '14715c06d.jpg',  '33e0ff2d5.jpg',\n                '4d4e09f2a.jpg', '877691df8.jpg', '8b909bb20.jpg', 'a8d99130e.jpg', \n                'ad55c3143.jpg', 'c8260c541.jpg', 'd6c7f17c7.jpg', 'dc3e7c901.jpg',\n                'e44dffe88.jpg', 'ef87bad36.jpg', 'f083256d8.jpg'] #corrupted images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1b6a04600ea9e44401515d292067af1b5ab9cf8","collapsed":true},"cell_type":"code","source":"nw = 2   #number of workers for data loader\narch = resnet34 #specify target architecture","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7491c8ef4a4be7d4d437d950d814497de7b4268"},"cell_type":"markdown","source":"### Data"},{"metadata":{"trusted":true,"_uuid":"1d2bbf7abe0b64932d0fd9d7497cdd7d3853881a","collapsed":true},"cell_type":"code","source":"train_names = [f for f in os.listdir(TRAIN)]\ntest_names = [f for f in os.listdir(TEST)]\nfor el in exclude_list:\n    if(el in train_names): train_names.remove(el)\n    if(el in test_names): test_names.remove(el)\n#5% of data in the validation set is sufficient for model evaluation\ntr_n, val_n = train_test_split(train_names, test_size=0.05, random_state=42)\nsegmentation_df = pd.read_csv(os.path.join(PATH, SEGMENTATION)).set_index('ImageId')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0e06d20508ecbd80b929dde5b3cd1f0ea6e10167"},"cell_type":"markdown","source":"As explained in https://www.kaggle.com/iafoss/unet34-dice-0-87/notebook, I drop all images without ships. The model responsible for ship detection will take care of them."},{"metadata":{"trusted":true,"_uuid":"4f7c4937cf0cb83bc419977b198bcb71739e39db"},"cell_type":"code","source":"def cut_empty(names):\n    return [name for name in names \n            if(type(segmentation_df.loc[name]['EncodedPixels']) != float)]\n\ntr_n_cut = cut_empty(tr_n)\nval_n_cut = cut_empty(val_n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98426b544b2ec283847b05900cdef8fef96cca26","collapsed":true},"cell_type":"code","source":"def get_mask(img_id, df):\n    shape = (768,768)\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    masks = df.loc[img_id]['EncodedPixels']\n    if(type(masks) == float): return img.reshape(shape)\n    if(type(masks) == str): masks = [masks]\n    for mask in masks:\n        s = mask.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1\n    return img.reshape(shape).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4412b5292790c30d77f8320115765541e670cab5","collapsed":true},"cell_type":"code","source":"class pdFilesDataset(FilesDataset):\n    def __init__(self, fnames, path, transform):\n        self.segmentation_df = pd.read_csv(SEGMENTATION).set_index('ImageId')\n        super().__init__(fnames, transform, path)\n    \n    def get_x(self, i):\n        img = open_image(os.path.join(self.path, self.fnames[i]))\n        if self.sz == 768: return img \n        else: return cv2.resize(img, (self.sz, self.sz))\n    \n    def get_y(self, i):\n        mask = np.zeros((768,768), dtype=np.uint8) if (self.path == TEST) \\\n            else get_mask(self.fnames[i], self.segmentation_df)\n        img = Image.fromarray(mask).resize((self.sz, self.sz)).convert('RGB')\n        return np.array(img).astype(np.float32)\n    \n    def get_c(self): return 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0b680f127a97227a96d0b890fb3f26c474ff0f5","collapsed":true},"cell_type":"code","source":"def get_data(sz,bs):\n    tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)\n    tr_names = tr_n if (len(tr_n_cut)%bs == 0) else tr_n[:-(len(tr_n_cut)%bs)] #cut incomplete batch\n    ds = ImageData.get_ds(pdFilesDataset, (tr_names,TRAIN), \n                (val_n_cut,TRAIN), tfms, test=(test_names,TEST))\n    md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)\n    md.is_multi = False\n    return md","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2c36d31bbf1a3afc3996222d450608f66bbb3d6"},"cell_type":"markdown","source":"### Model"},{"metadata":{"trusted":true,"_uuid":"785eb13d12159b2a90835e248093f2e4bd3d661a","collapsed":true},"cell_type":"code","source":"cut,lr_cut = model_meta[arch]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e9c3d8cdad5eaccbc2543a6f4c7c7b498c03888","collapsed":true},"cell_type":"code","source":"def get_base():                   #load ResNet34 model\n    layers = cut_model(arch(True), cut)\n    return nn.Sequential(*layers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8147f86421b467087ec60dbddd0f1b88e88844cd","collapsed":true},"cell_type":"code","source":"class UnetBlock(nn.Module):\n    def __init__(self, up_in, x_in, n_out):\n        super().__init__()\n        up_out = x_out = n_out//2\n        self.x_conv  = nn.Conv2d(x_in,  x_out,  1)\n        self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)\n        self.bn = nn.BatchNorm2d(n_out)\n        \n    def forward(self, up_p, x_p):\n        up_p = self.tr_conv(up_p)\n        x_p = self.x_conv(x_p)\n        cat_p = torch.cat([up_p,x_p], dim=1)\n        return self.bn(F.relu(cat_p))\n\nclass SaveFeatures():\n    features=None\n    def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)\n    def hook_fn(self, module, input, output): self.features = output\n    def remove(self): self.hook.remove()\n    \nclass Unet34(nn.Module):\n    def __init__(self, rn):\n        super().__init__()\n        self.rn = rn\n        self.sfs = [SaveFeatures(rn[i]) for i in [2,4,5,6]]\n        self.up1 = UnetBlock(512,256,256)\n        self.up2 = UnetBlock(256,128,256)\n        self.up3 = UnetBlock(256,64,256)\n        self.up4 = UnetBlock(256,64,256)\n        self.up5 = nn.ConvTranspose2d(256, 1, 2, stride=2)\n        \n    def forward(self,x):\n        x = F.relu(self.rn(x))\n        x = self.up1(x, self.sfs[3].features)\n        x = self.up2(x, self.sfs[2].features)\n        x = self.up3(x, self.sfs[1].features)\n        x = self.up4(x, self.sfs[0].features)\n        x = self.up5(x)\n        return x[:,0]\n    \n    def close(self):\n        for sf in self.sfs: sf.remove()\n            \nclass UnetModel():\n    def __init__(self,model,name='Unet'):\n        self.model,self.name = model,name\n\n    def get_layer_groups(self, precompute):\n        lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))\n        return lgs + [children(self.model)[1:]]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8de38bae09beab2d2acd7c824373cbd4b111309b"},"cell_type":"markdown","source":"### Score evaluation"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"8778da2e4fd46ccca6a551cc84a022f4c3c18140"},"cell_type":"code","source":"def IoU(pred, targs):\n    pred = (pred > 0.5).astype(float)\n    intersection = (pred*targs).sum()\n    return intersection / ((pred+targs).sum() - intersection + 1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"deee9be6cad6c5dde352e1022837e50550fa5a3c"},"cell_type":"code","source":"def get_score(pred, true):\n    n_th = 10\n    b = 4\n    thresholds = [0.5 + 0.05*i for i in range(n_th)]\n    n_masks = len(true)\n    n_pred = len(pred)\n    ious = []\n    score = 0\n    for mask in true:\n        buf = []\n        for p in pred: buf.append(IoU(p,mask))\n        ious.append(buf)\n    for t in thresholds:   \n        tp, fp, fn = 0, 0, 0\n        for i in range(n_masks):\n            match = False\n            for j in range(n_pred):\n                if ious[i][j] > t: match = True\n            if not match: fn += 1\n        \n        for j in range(n_pred):\n            match = False\n            for i in range(n_masks):\n                if ious[i][j] > t: match = True\n            if match: tp += 1\n            else: fp += 1\n        score += ((b+1)*tp)/((b+1)*tp + b*fn + fp)       \n    return score/n_th","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7e7ea344cecf23394740a013142b26320092b5c6"},"cell_type":"markdown","source":"In this competition we should submit and individual mask for each identified ship. The simplest way to do it is splitting the total mask into individual ones based on the connectivity of detected objects."},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"23628b68164333efe7bb09b9690bc0ccd4138984"},"cell_type":"code","source":"def split_mask(mask):\n    threshold = 0.5\n    threshold_obj = 8 #ignor predictions composed of \"threshold_obj\" pixels or less\n    labled,n_objs = ndimage.label(mask > threshold)\n    result = []\n    for i in range(n_objs):\n        obj = (labled == i + 1).astype(int)\n        if(obj.sum() > threshold_obj): result.append(obj)\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c0b03496a63d7115b005aad9b96ee40420662c61"},"cell_type":"code","source":"def get_mask_ind(img_id, df, shape = (768,768)): #return mask for each ship\n    masks = df.loc[img_id]['EncodedPixels']\n    if(type(masks) == float): return []\n    if(type(masks) == str): masks = [masks]\n    result = []\n    for mask in masks:\n        img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n        s = mask.split()\n        for i in range(len(s)//2):\n            start = int(s[2*i]) - 1\n            length = int(s[2*i+1])\n            img[start:start+length] = 1\n        result.append(img.reshape(shape).T)\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d1c265c22c8c56ae3b4b00dbd7fc6f4761d66158"},"cell_type":"code","source":"class Score_eval():\n    def __init__(self):\n        self.segmentation_df = pd.read_csv(SEGMENTATION).set_index('ImageId')\n        self.score, self.count = 0.0, 0\n        \n    def put(self,pred,name):\n        true = get_mask_ind(name, self.segmentation_df)\n        self.score += get_score(pred,true)\n        self.count += 1\n        \n    def evaluate(self):\n        return self.score/self.count","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fa7634b20774b7c2293d5f15f6f37ab2e19f85a2"},"cell_type":"markdown","source":"### Prediction"},{"metadata":{"trusted":true,"_uuid":"c9700cc48f14aed6ef66024dccb5da51ab5df004"},"cell_type":"code","source":"m = to_gpu(Unet34(get_base()))\nmodels = UnetModel(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"741a0f34d5da2ac0a97f88a32685a10d5771950e","collapsed":true},"cell_type":"code","source":"sz = 768 #image size\nbs = 8  #batch size\nmd = get_data(sz,bs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cb66dcc8fb4a8ee7d583277199e6ca52b9a094a"},"cell_type":"code","source":"learn = ConvLearner(md, models)\nlearn.models_path = PRETRAINED_SEGMENTATION_PATH\nlearn.load('Unet34_768_1')\nlearn.models_path = PATH","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2338c12e233fd9001a6e959b18deb527ecec9e99"},"cell_type":"markdown","source":"Since the model predicts pixel masks, which are quite large, running standard functions for making a prediction will fail due to memory issue, especially for the test set, where about 100k 786x786 pixel masks should be created. Therefore, I wrote a function that does prediction batch by batch and applies F_save function for each generated mask."},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4b5dd41b4af5ad28a13328b1a41f695f760010a7"},"cell_type":"code","source":"def model_pred(learner, dl, F_save): #if use train dl, disable shuffling\n    learner.model.eval();\n    name_list = dl.dataset.fnames\n    num_batchs = len(dl)\n    t = tqdm(iter(dl), leave=False, total=num_batchs)\n    count = 0\n    for x,y in t:\n        py = to_np(F.sigmoid(learn.model(V(x))))\n        batch_size = len(py)\n        for i in range(batch_size):\n            F_save(py[i],to_np(y[i]),name_list[count])\n            count += 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"509e3579f27dfd404b575f8475a35a50759aef41"},"cell_type":"markdown","source":"Running the model evaluation on the validation set."},{"metadata":{"trusted":true,"_uuid":"b1aa3ec5340bbb698142cbb89755ba5e631d6bb1"},"cell_type":"code","source":"score = Score_eval()\nprocess_pred = lambda yp, y, name : score.put(split_mask(yp),name)\nmodel_pred(learn, md.val_dl, process_pred)\nprint('\\n',score.evaluate())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5b2b0ed553655a84761ac02bf3894a8d6703d344"},"cell_type":"markdown","source":"It is the **score based only on images with ships**, a model responsible for ship detection (accuracy ~98%) takes care of images without ships. Since the fraction of empty images in the test set is ~0.85, the expected score of the model stacked with ship detection one (https://www.kaggle.com/iafoss/fine-tuning-resnet34-on-ship-detection/notebook) is approximately 0.85 + 0.34 * 0.15 = 0.90. However, you should keep in mind that the evaluated model has been trained only for one epoch on full resolution images (the dice is only ~0.80 for 784x784 images). I tried to do similar testing for a model with dice 0.895 and got 0.44 based on images only with ships that would result in 0.92 for the model evaluation score. Continuing training the model, TTA, and mask postprocessing can further boost it."},{"metadata":{"_uuid":"a4308d215ee4970ebf0ce80964b826c9fa4bf630"},"cell_type":"markdown","source":"### Submission"},{"metadata":{"_uuid":"e13243a1755ab175be3d3968076d34f01bb86e41"},"cell_type":"markdown","source":"Load the prediction of ship detection model (https://www.kaggle.com/iafoss/fine-tuning-resnet34-on-ship-detection/notebook) for the test set."},{"metadata":{"trusted":true,"_uuid":"4060cbb4a43de4899dbab4ed2f4cf3d557431ed9"},"cell_type":"code","source":"ship_detection = pd.read_csv(DETECTION_TEST_PRED)\nship_detection.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"95bb6b7a994acc27454d192d1bc7bb9de8a41876"},"cell_type":"markdown","source":"Identify images with ships and run Unet34 model only for them (~15%)."},{"metadata":{"trusted":true,"_uuid":"f0923fdae18aceffcdbf6b067e2e78bf794e1cb8"},"cell_type":"code","source":"test_names = ship_detection.loc[ship_detection['p_ship'] > 0.5, ['id']]['id'].values.tolist()\ntest_names_nothing = ship_detection.loc[ship_detection['p_ship'] <= 0.5, ['id']]['id'].values.tolist()\nlen(test_names), len(test_names_nothing)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e8b37e28cc5dc23d6769aa0765eaeadb4df9a0de"},"cell_type":"code","source":"md = get_data(sz,bs)\nlearn.set_data(md)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8bb0a20538d9e51a3c1efbeaecbe064d7653f863"},"cell_type":"markdown","source":"The function for mask decoding is borrowed from https://www.kaggle.com/kmader/from-trained-u-net-to-submission-part-2/notebook ."},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ec3a58469656ee1b7040d37ba36f6f1344753767"},"cell_type":"code","source":"def decode_mask(mask, shape=(768, 768)):\n    pixels = mask.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"227ebf463005e664a9e8c0728b7aeb2c69765c60"},"cell_type":"code","source":"ship_list_dict = []\nfor name in test_names_nothing:\n    ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1e9f634e2e81c766c27179234fd843a73b35bf0c"},"cell_type":"code","source":"def enc_test(yp, y, name):\n    masks = split_mask(yp)\n    if(len(masks) == 0): \n        ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})\n    for mask in masks:\n        ship_list_dict.append({'ImageId':name,'EncodedPixels':decode_mask(mask)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e311f443432e40efdfb6b6a55c3891689f6aa2b3"},"cell_type":"code","source":"model_pred(learn, md.test_dl, enc_test)\npred_df = pd.DataFrame(ship_list_dict)\npred_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}