{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Imports and Data preperation"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n!pip install git+https://github.com/albumentations-team/albumentations.git\n!pip install git+https://github.com/qubvel/segmentation_models.pytorch\n\nimport albumentations as ag\nimport matplotlib.pyplot as plt\nimport skimage.data as skd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom segmentation_models_pytorch.encoders import get_preprocessing_fn\nimport segmentation_models_pytorch as smp\nimport tqdm\nimport cv2 \nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\n# create datasets\ntrain_path = \"../input/airbus-ship-detection/train_v2\"\ntest_path = \"../input/airbus-ship-detection/test_v2\"\ntrain_imagelist = os.listdir(train_path)\ntest_imagelist = os.listdir(test_path)\n\n\nmasks = pd.read_csv('../input/airbus-ship-detection/train_ship_segmentations_v2.csv')\n\n\nimg_masks = masks[masks['EncodedPixels'].notnull()]['ImageId'].tolist() # get none empty\nimg_masks_noships = masks[masks['EncodedPixels'].isnull()]['ImageId'].tolist() # get none empty\n# masks = masks.iloc[img_masks] # set as training\n\n\n# p1 = 8172\np1 = 7092\nmasks['ImageId'] = train_path + \"/\" + masks['ImageId'].astype(str)\n\nhasships_paths = masks[masks['EncodedPixels'].notnull()]['ImageId'].tolist()        #[train_path + \"/\" + name  for name in tqdm.notebook.tqdm(train_imagelist) if name in img_masks] \nnoships_paths =  masks[masks['EncodedPixels'].isnull()]['ImageId'].tolist() #[train_path + \"/\" + name  for name in tqdm.notebook.tqdm(train_imagelist) if name in img_masks_noships]\n        \nall_paths_train = hasships_paths #+ noships_paths\n\n\nprint(len(all_paths_train), \" samples\")\n\nvalidx = all_paths_train[:p1]\ntestx = all_paths_train[p1:2*p1]\ntrainx = all_paths_train[2*p1:] \n\nnum_hweight = len(hasships_paths) - 2*p1\nnum_nhweight = len(trainx) - num_hweight\n\nhweight = 1.0/num_hweight\nnhweight = 0 #0.0/num_nhweight\n\n# prepare submission\nsubx = [test_path + \"/\" + name for name in test_imagelist]\n\nprint(f\"training: {len(trainx)} validation {len(validx)} test {len(testx)}\")\n\nmasks.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"number of images with ships: {len(hasships_paths)}\")\nprint(f\"number of images without ships: {len(noships_paths)}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Usefull Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_decode(mask_rle, shape=(768, 768)):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_tensor(x, **kwargs):\n    # return x.transpose(2, 0, 1).astype('float32')\n    return x.transpose(2, 1, 0).astype('float32')\n    # return np.rollaxis(x, 2, 0).astype('float32') # other option\n  \ndef get_preprocessing(preprocessing_fn):\n    \"\"\"Construct preprocessing transform\n    \n    Args:\n        preprocessing_fn (callbale): data normalization function \n            (can be specific for each pretrained neural network)\n    Return:\n        transform: albumentations.Compose\n    \n    \"\"\"\n    \n    _transform = [\n        ag.augmentations.transforms.Lambda(image=preprocessing_fn),\n        ag.augmentations.transforms.Lambda(image=to_tensor, mask=to_tensor),\n    ]\n    return ag.Compose(_transform)\n\nclass Dataset_ships(Dataset):\n  def __init__(self, x, y, transforms = None, preprocessing = None):\n\n    self.x = x\n    self.y = y\n    self.transforms = transforms\n    self.preprocessing = preprocessing\n    \n\n  def __getitem__ (self, idx):\n    \n    \n    device = None\n    if torch.cuda.is_available():\n      device = torch.device(\"cuda\")          # a CUDA device object\n    \n    x = cv2.imread(self.x[idx])\n    x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB)\n    \n    # Take the individual ship masks and create a single mask array for all ships\n    ImageId = self.x[idx]#.split(\"/\")[-1]\n    img_masks = self.y.loc[self.y['ImageId'] == ImageId, 'EncodedPixels'].tolist()\n \n    y = np.zeros((768, 768))\n    \n\n    if isinstance(img_masks[0], str):\n      for mask in img_masks:\n          y += rle_decode(mask)\n\n#     y = ag.Resize(224, 224, p=1)(image=y)['image']\n\n    if self.transforms != None:\n      transform = self.transforms(image=x, mask=y)\n      x = transform['image']\n      y = transform['mask']\n\n    y = np.expand_dims(y, 2).astype(np.byte)\n\n    if self.preprocessing != None:\n      sample = self.preprocessing(image=x, mask=y)\n      x = sample['image']\n      y = sample['mask']\n\n\n    y = torch.tensor(y, device=device)\n    x = torch.tensor(x, device=device)\n        \n    return x, y\n\n  def __len__(self):\n    # return len(self.x)\n    return len(self.x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from segmentation_models_pytorch.utils.meter import AverageValueMeter\nimport sys\n\ndef run_epoch(epoch, dataloader, log_frequncy=100, threshold=None):\n\n        epoch.on_epoch_start()\n        \n        logs = {}\n        loss_meter = AverageValueMeter()\n        metrics_meters = {metric.__name__: AverageValueMeter() for metric in epoch.metrics}\n        per_batch_metrics = {metric_fn.__name__:[] for metric_fn in epoch.metrics} # create item to track metrics per batch\n        per_batch_metrics.update({'loss': []})\n\n        n = 0\n\n        with tqdm.notebook.tqdm(dataloader, desc=epoch.stage_name, file=sys.stdout, disable=not (epoch.verbose)) as iterator:\n            print(f\"len iterator {len(iterator)}\")\n            for x, y in iterator:\n                x, y = x.to(epoch.device), y.to(epoch.device)\n                loss, y_pred = epoch.batch_update(x, y)\n                \n                if threshold is not None:\n                    y_pred = (y_pred > threshold) * y_pred\n                    y_pred[y_pred!=0] = 1\n                \n                # update loss logs\n                loss_value = loss.cpu().detach().numpy()\n                loss_meter.add(loss_value)\n                loss_logs = {epoch.loss.__name__: loss_meter.mean}\n                logs.update(loss_logs)\n\n                # update metrics logs\n                for metric_fn in epoch.metrics:\n                    metric_value = metric_fn(y_pred, y).cpu().detach().numpy()\n                    metrics_meters[metric_fn.__name__].add(metric_value)\n\n                    if n % log_frequncy == 0:\n                      per_batch_metrics[metric_fn.__name__].append(metric_value)\n\n                if n % log_frequncy == 0:\n                  per_batch_metrics['loss'].append(loss_value)\n\n                metrics_logs = {k: v.mean for k, v in metrics_meters.items()}\n                logs.update(metrics_logs)\n\n                if epoch.verbose:\n                    s = epoch._format_logs(logs)\n                    iterator.set_postfix_str(s)\n\n                n += 1\n\n        return logs, per_batch_metrics\n\n\n\n# todo add arguments\ndef train_model(m, critation, optimizer, train_loader, valid_loader, test_dataloader, epoches=2, log_frequncy=100, scheduler=None):\n  device = None\n  if torch.cuda.is_available():\n    device = torch.device(\"cuda\")          # a CUDA device object\n\n  if device != None:\n    m.cuda()\n\n  print(f\"device: {device}, {next(m.parameters()).is_cuda }\")\n\n  metrics = [\n      smp.utils.metrics.IoU(threshold=0.5),\n      smp.utils.metrics.Accuracy(),\n      smp.utils.metrics.Recall(),\n      smp.utils.metrics.Precision(),\n      smp.utils.metrics.Fscore()\n  ]\n\n  train_epoch = smp.utils.train.TrainEpoch(\n      m, \n      loss=critation, \n      metrics=metrics, \n      optimizer=optimizer,\n      device=device,\n      verbose=True,\n  )\n\n  valid_epoch = smp.utils.train.ValidEpoch(\n      m, \n      loss=critation, \n      metrics=metrics, \n      device=device,\n      verbose=True,\n  )\n\n  # train model for some epochs\n\n  max_score = 0\n  train_logs = []\n  valid_logs = []\n  train_metrics_logs = []\n\n  for i in range(0, epoches):\n    \n    print('\\nEpoch: {}'.format(i))\n    # train_logs.append(train_epoch.run(train_loader)) # the lines beneth should replace this\n    # valid_logs.append(valid_epoch.run(valid_loader))\n    \n    # todo test to see if works\n    train_log, train_metrics = run_epoch(train_epoch, train_loader, log_frequncy)\n    valid_log, valid_metrics = run_epoch(valid_epoch, valid_loader, log_frequncy)\n    train_metrics_logs.append(train_metrics)\n    train_logs.append(train_log)\n    valid_logs.append(valid_log)\n\n\n  # evaluate model on test set\n  test_epoch = smp.utils.train.ValidEpoch(\n      model=m,\n      loss=critation,\n      metrics=metrics,\n      device=device,\n  )\n    \n  test_logs = test_epoch.run(test_dataloader)\n    \n  return [train_logs, valid_logs, test_logs], train_metrics_logs\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_examples(model, loader, num, threshold = None, ensemble=False):\n\n  itr = iter(loader)\n  x, t = next(itr)\n\n  xs = []\n  ys = []\n  ts = []\n  if ensemble:\n    for m in model:\n        m.eval()\n  else:\n    model.eval()\n  with torch.no_grad():\n    \n    for j in range(num):\n      x, t = next(itr)\n    \n\n      y = model(x)\n      y = np.array(y.cpu()).squeeze()\n\n      x = np.array(x.cpu()).squeeze()\n      x = np.rollaxis(x,0,3)\n      x = x + np.abs(x.min())\n      x = (x/x.max())\n\n      \n      t = np.array(t.cpu()).squeeze()\n      \n      if threshold != None:\n        y = (y > threshold) * y\n        y[y!=0] = 1\n    \n      xs.append(x)\n      ys.append(y)\n      ts.append(t)\n\n    fig, axarr = plt.subplots(num, 4, figsize=(20, 45))\n    fig.tight_layout()\n    for i in range(0,num):\n      \n      axarr[i][0].axis('off')\n      axarr[i][1].axis('off')\n      axarr[i][2].axis('off')\n      axarr[i][3].axis('off')\n\n      axarr[i][0].set_title(\"image\")\n      axarr[i][1].set_title(\"detection\")\n      axarr[i][2].set_title(\"ground truth\")\n      axarr[i][3].set_title(\"overlay\")\n\n      axarr[i][0].imshow(xs[i])\n      axarr[i][1].imshow(ys[i])\n      axarr[i][2].imshow(ts[i])\n      axarr[i][3].imshow(xs[i])\n      axarr[i][3].imshow(ys[i], alpha=0.4)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset_ships_submission(Dataset):\n  def __init__(self, x, y, transforms = None, preprocessing = None):\n\n    self.x = x\n    self.y = y\n    self.transforms = transforms\n    self.preprocessing = preprocessing\n\n  def __getitem__ (self, idx):\n    \n    device = None\n    if torch.cuda.is_available():\n      device = torch.device(\"cuda\")          # a CUDA device object\n    \n    x = cv2.imread(self.x[idx])\n    x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB)\n    \n    # Take the individual ship masks and create a single mask array for all ships\n    ImageId = self.x[idx].split(\"/\")[-1]\n\n    if self.transforms != None:\n      transform = self.transforms(image=x)\n      x = transform['image']\n\n    if self.preprocessing != None:\n      sample = self.preprocessing(image=x)\n      x = sample['image']\n\n    x = torch.tensor(x, device=device)\n        \n    return x, ImageId\n\n  def __len__(self):\n    # return len(self.x)\n    return len(self.x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_encode(img, min_max_threshold=1e-3, max_mean_threshold=None):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    if np.max(img) < min_max_threshold:\n        return '' ## no need to encode if it's all zeros\n    if max_mean_threshold and np.mean(img) > max_mean_threshold:\n        return '' ## ignore overfilled mask\n    pixels = img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = torch.load('../input/airbus-ships-weights-danielk/linkcheckpoint0.31.pth.tar', map_location=torch.device('cuda'))\nprint(checkpoint.keys())\nmodel = checkpoint['model']\noptimizer = checkpoint['optimizer']\ncritation = checkpoint['critation']\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training Loop"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"ENCODER = 'resnet101'\nENCODER_WEIGHTS = 'imagenet'\nCLASSES = ['ship']\nACTIVATION = None # could be None for logits or 'softmax2d' for multicalss segmentation\nDEVICE = 'cuda'\n\nimport os, ssl\nif (not os.environ.get('PYTHONHTTPSVERIFY', '') and\ngetattr(ssl, '_create_unverified_context', None)):\n    ssl._create_default_https_context = ssl._create_unverified_context\n\nmodel = smp.Linknet(\n    encoder_name=ENCODER, \n    # encoder_weights=ENCODER_WEIGHTS, \n    classes=len(CLASSES), \n    activation=ACTIVATION,\n)\n\nmodel = model.cuda()\n\nagu = ag.Compose([ag.RandomBrightness(p=0.5),ag.RandomRotate90(p=0.5), ag.RandomCrop(256,256,p=1)], p=1)\nagu_valid = ag.Compose([], p=1)\n\npreprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER)#, ENCODER_WEIGHTS)\npreprocess = get_preprocessing(preprocessing_fn)\n\ndataset_train = Dataset_ships(trainx, masks, agu, preprocess)\ntrain_loader = DataLoader(dataset_train, batch_size=32, shuffle=True) # sampler=weighted_sampler)\n\ndataset_valid = Dataset_ships(validx, masks, None, preprocess)\nvalid_loader = DataLoader(dataset_valid, batch_size=32, shuffle=True)\n\ndataset_test = Dataset_ships(testx, masks, None, preprocess)\ntest_dataloader = DataLoader(dataset_test, batch_size=1, shuffle=True)\n\ncritation = smp.utils.losses.BCEWithLogitsLoss()\noptimizer = torch.optim.Adam([\n      dict(params=model.parameters(), lr=0.0015),\n  ])\n\n\n\nmodel = model.cuda()\naverage_per_epoch_logs, train_metrics_logs = train_model(model, critation, optimizer, train_loader, valid_loader, test_dataloader, 2, 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"***Saving model state***"},{"metadata":{"trusted":true},"cell_type":"code","source":"state = {\n             'model': model,\n             'optimizer': optimizer,\n             'critation':critation}\nfilename = 'model.pth.tar'\ntorch.save(state, filename)\n\ntorch.save(model, \"model.pth\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation Loop on a trained model"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"\nENCODER = 'resnet101'\n\nmodel = torch.load(\"../input/airbus-ships-weights-danielk/checkpoint0.77_unet.pth.tar\")['model']\n\ncritation = smp.utils.losses.BCEWithLogitsLoss()\n\npreprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER) #, ENCODER_WEIGHTS)\npreprocess = get_preprocessing(preprocessing_fn)\n\ndataset_valid = Dataset_ships(validx, masks, None, preprocess)\nvalid_loader = DataLoader(dataset_valid, batch_size=16, shuffle=True)\n\ndataset_test = Dataset_ships(testx, masks, None, preprocess)\ntest_dataloader = DataLoader(dataset_test, batch_size=16, shuffle=True)\n\nmetrics = [\n     smp.utils.metrics.IoU(threshold=0.5),\n     smp.utils.metrics.Accuracy(),\n     smp.utils.metrics.Recall(),\n     smp.utils.metrics.Precision(),\n     smp.utils.metrics.Fscore()\n]\n\nvalid_epoch = smp.utils.train.ValidEpoch(\n      model,\n      loss=critation,\n      metrics=metrics,\n      device='cuda',\n      verbose=True,\n  )\n\nmodel.to('cuda')\nmodel.eval\nvalid_log, valid_metrics = run_epoch(valid_epoch, test_dataloader, 20, threshold=None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# View Training graphs"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_patch_graphs = [j for kk in train_metrics_logs for j in kk['loss'] ]\ntrain_patch_graphs2 = [j for kk in train_metrics_logs for j in kk['accuracy'] ]\ntrain_patch_graphs3 = [j for kk in train_metrics_logs for j in kk['iou_score'] ]\ntrain_patch_graphs4 = [j for kk in train_metrics_logs for j in kk['fscore'] ]\ntrain_patch_graphs5 = [j for kk in train_metrics_logs for j in kk['recall'] ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(train_patch_graphs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(train_patch_graphs4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(train_patch_graphs3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(train_patch_graphs2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ensemble Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = torch.load('../input/airbus-ships-weights-danielk/linkcheckpoint0.48.pth.tar', map_location=torch.device('cuda'))\nprint(checkpoint.keys())\nmodel1 = checkpoint['model']\nmodel1 = model1.cuda()\nmodel1 = model1.eval()\n\ncheckpoint = torch.load('../input/airbus-ships-weights-danielk/checkpoint0.77_unet.pth.tar', map_location=torch.device('cuda'))\nprint(checkpoint.keys())\nmodel2 = checkpoint['model']\nmodel2 = model2.cuda()\nmodel2 = model2.eval()\n\ncheckpoint = torch.load('../input/airbus-ships-weights-danielk/checkpoint0.67_new.pth.tar', map_location=torch.device('cuda'))\nprint(checkpoint.keys())\nmodel3 = checkpoint['model']\nmodel3 = model3.cuda()\nmodel3 = model3.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ensemble(torch.nn.Module):\n    \n    def __init__(self, models, threshold=None, late_threshold=None):\n        super().__init__()\n        self.models = models\n        self.threshold = threshold\n        self.late_threshold = late_threshold\n\n    def forward(self, x):\n        y = self.models[0](x)\n        \n        if self.threshold is not None:\n            y = (y > self.threshold[0]) * y\n            y[y!=0] = 1\n        \n        for i in range(1,len(self.models)):\n            \n            cy = self.models[i](x)\n            \n            if self.threshold is not None:\n              cy = (cy > self.threshold[i]) * cy\n              cy[cy!=0] = 1\n            \n            y = y + cy\n            \n        if self.threshold is not None:\n            y[y > len(self.models)/2] = 1\n            y[y!=0]=1\n            \n        elif self.late_threshold is not None:\n             y = y/len(self.models)\n             y = (y > self.late_threshold) * y\n             y[y!=0] = 1\n                \n        else:\n            y = y/len(self.models)\n            \n        \n        return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#showing examples\nENCODER = 'resnet101'\n\n# enmodel = ensemble([model, model2, model3], None, 0)  # uncomment for average ensemble\nenmodel = ensemble([model, model2, model3], [0,0,0])  # uncomment to enable majority vote ensemble \n\n# enmodel = enmodel.cuda()\n\npreprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER)#, ENCODER_WEIGHTS)\npreprocess = get_preprocessing(preprocessing_fn)\n\nagu = ag.Compose([ag.RandomBrightnessContrast(p=1) ,ag.Rotate(p=0.5)], p=1)\n\ndataset_test = Dataset_ships(testx, masks, None, preprocess)\ntest_dataloader = DataLoader(dataset_test, batch_size=1, shuffle=True)\n\n# model.cuda()\nshow_examples(enmodel, test_dataloader, 15, threshold=None, ensemble=False)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Validation on Ensemble Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nENCODER = 'resnet101'\n\nmodel = torch.load(\"../input/airbus-ships-weights-danielk/checkpoint0.77_unet.pth.tar\")['model']\n\ncritation = smp.utils.losses.BCEWithLogitsLoss()\n\npreprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER) #, ENCODER_WEIGHTS)\npreprocess = get_preprocessing(preprocessing_fn)\n\ndataset_valid = Dataset_ships(validx, masks, None, preprocess)\nvalid_loader = DataLoader(dataset_valid, batch_size=16, shuffle=True)\n\ndataset_test = Dataset_ships(testx, masks, None, preprocess)\ntest_dataloader = DataLoader(dataset_test, batch_size=16, shuffle=True)\n\nmetrics = [\n     smp.utils.metrics.IoU(threshold=0.5),\n     smp.utils.metrics.Accuracy(),\n     smp.utils.metrics.Recall(),\n     smp.utils.metrics.Precision(),\n     smp.utils.metrics.Fscore()\n]\n\nvalid_epoch = smp.utils.train.ValidEpoch(\n      enmodel,\n      loss=critation,\n      metrics=metrics,\n      device='cuda',\n      verbose=True,\n  )\n\nmodel.to('cuda')\nmodel.eval\nvalid_log, valid_metrics = run_epoch(valid_epoch, test_dataloader, 20, threshold=None)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission "},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"subx = [test_path + \"/\" + name for name in test_imagelist]\nENCODER = 'resnet101'\n\npreprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER)#, ENCODER_WEIGHTS)\npreprocess = get_preprocessing(preprocessing_fn)\n\ndataset_test = Dataset_ships_submission(subx, masks, None, preprocess)\ntest_dataloader = DataLoader(dataset_test, batch_size=1, shuffle=True)\n\n# show_examples(model, test_dataloader, 3)\nitr = iter(test_dataloader)\n\nout_pred_rows = []\n\nenmodel.eval()\nfor i in tqdm.notebook.tqdm(range(len(itr))):\n  x, imgid = next(itr)\n  imgid = imgid[0]\n\n  enmodel.to('cuda')\n  y = enmodel(x)\n  y = y.cpu().detach().numpy().squeeze()\n\n  c_rle = rle_encode(y)\n  out_pred_rows += [{'ImageId': imgid, 'EncodedPixels': c_rle}]\n\nsubmission_df = pd.DataFrame(out_pred_rows)[['ImageId', 'EncodedPixels']]\nsubmission_df.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_test = Dataset_ships(testx, masks, None, preprocess)\ntest_dataloader = DataLoader(dataset_test, batch_size=1, shuffle=True)\nshow_examples(enmodel,test_dataloader,10, None)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}