{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Outline"},{"metadata":{},"cell_type":"markdown","source":"- Installation\n- Import\n- Dataset\n- Training\n- Post Processing\n- Submission"},{"metadata":{},"cell_type":"markdown","source":"Break down the results of your classifier.\n - Which category is the easiest to classify? And the hardest? Did this correlate with the distribution of training data? \n - Visualize five examples where classification went well, and five where classification failed, consider what went wrong in case of the latter. \n\nWrite a short tech report that gives a deeper explanation of your experiments and the rationales. \n - Hyper-parameters\n - Batch sizes\n - Number of network layers \n - Adding/removing batch normalization \n - Increasing/decreasing dropout (if applicable) \n - Data augmentation techniques (random cropping, normalization, random erasing, etc.) \n"},{"metadata":{},"cell_type":"markdown","source":"# Installation"},{"metadata":{"trusted":true},"cell_type":"code","source":"#!pip install torch\n#!pip install torchvision\n#!pip install tqdm\n#!pip install matplotlib\n#!pip install numpy\n#!pip install cv2\n#!pip install pandas\n\n!pip install draugr -U\n!pip install neodroidvision -U","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nfrom matplotlib import pyplot\nimport pandas\nimport seaborn\nimport torch\nimport numpy\nfrom pathlib import Path\n\nfrom neodroidvision import PROJECT_APP_PATH\nfrom neodroidvision.multitask.fission_net.skip_hourglass import SkipHourglassFissionNet\nfrom neodroidvision.segmentation import BCEDiceLoss, bool_dice, draw_convex_hull\nfrom neodroidvision.segmentation import mask_to_run_length\n\nimport draugr\nfrom draugr.torch_utilities import torch_seed, global_torch_device, float_chw_to_hwc_uint, chw_to_hwc, resize_image_cv\n\nimport albumentations\nimport cv2\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import Dataset, DataLoader\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Hyperparameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"pyplot.style.use('bmh')\n\nbase_data_path = Path.cwd().parent / 'input'\nbase_dataset_path = base_data_path / 'understanding_cloud_organization'\nimage_path = base_data_path / 'understanding-clouds-resized'\n\nsave_model_path = PROJECT_APP_PATH.user_data / 'cloud_seg.model'\n\nSEED = 87539842\nbatch_size = 8\nnum_workers = 2\ntorch_seed(SEED)\ncriterion = BCEDiceLoss(eps=1.0)\nlr = 3e-3\nencoding_depth=5\nn_epochs=30\nworking_mask_size = (640, 320) # divisible by 32, scales better with Unet architectures\nfinal_mask_size=(525,350)\nfinal_mask_size_T=final_mask_size[::-1]\nMIN_SIZES=[0, 100, 1200, 5000, 10000, 30000]\nthreshold_samples=20","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset"},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class CloudSegmentationDataset(Dataset):\n  categories = {0:\"Fish\", 1:\"Flower\", 2:\"Gravel\", 3:\"Sugar\"}\n  image_size = working_mask_size\n  image_size_T = image_size[::-1]\n\n  predictor_channels = 3\n  response_channels = len(categories)\n\n  predictors_shape = (*image_size_T, predictor_channels)\n  response_shape = (*image_size_T, response_channels)\n\n  predictors_shape_T = predictors_shape[::-1]\n  response_shape_T = response_shape[::-1]\n\n  mean = (0.2606705, 0.27866408, 0.32657165)  # Computed prior\n  std = (0.25366131, 0.24921637, 0.23504028)  # Computed prior\n\n  def training_augmentations(self):\n    return [albumentations.VerticalFlip(p=0.5),\n            albumentations.HorizontalFlip(p=0.5),\n            albumentations.ShiftScaleRotate(p=0.5,\n                                            rotate_limit=0,\n                                            border_mode=0\n                                            ),\n            ]\n\n  def validation_augmentations(self):\n    \"\"\"Add paddings to make image shape divisible by 32\"\"\"\n    return [\n      albumentations.Resize(*self.image_size_T),\n      # albumentations.Normalize(mean=self.mean, std=self.std)\n      # Standardization\n      ]\n\n  '''\n  def un_standardise(self, img):\n    \"\"\"Add paddings to make image shape divisible by 32\"\"\"\n    return (img * self.std + self.mean).astype(numpy.uint8)\n  '''\n\n  def __init__(self,\n               csv_path: Path,\n               image_data_path: Path,\n               subset: str = \"train\",\n               transp=True,\n               N_FOLDS=10,\n               SEED=246232,\n               ):\n\n    self.transp = transp\n\n    if subset != 'test':\n      data_frame = pandas.read_csv(csv_path / f'train.csv')\n    else:\n      data_frame = pandas.read_csv(csv_path / f'sample_submission.csv')\n\n    data_frame[\"label\"] = data_frame[\"Image_Label\"].apply(lambda x:x.split(\"_\")[1])\n    data_frame[\"im_id\"] = data_frame[\"Image_Label\"].apply(lambda x:x.split(\"_\")[0])\n    self.data_frame = data_frame\n    self.subset = subset\n    self.base_image_data = image_data_path\n\n    if subset != 'test':\n      id_mask_count = (data_frame.loc[data_frame[\"EncodedPixels\"].isnull() == False, \"Image_Label\"]\n                       .apply(lambda x:x.split(\"_\")[0])\n                       .value_counts()\n                       .sort_index()\n                       .reset_index()\n                       .rename(columns={\"index\":\"img_id\", \"Image_Label\":\"count\"})\n                       )  # split data into train and val\n\n      ids = id_mask_count[\"img_id\"].values\n      li = [[train_index, test_index]\n            for train_index, test_index\n            in StratifiedKFold(n_splits=N_FOLDS,\n                               random_state=SEED\n                               ).split(ids, id_mask_count[\"count\"])\n            ]\n\n      self.image_data_path = image_data_path / 'train_images_525'/'train_images_525'\n\n      if subset == 'valid':\n        self.img_ids = ids[li[0][1]]\n      else:\n        self.img_ids = ids[li[0][0]]\n    else:\n      self.img_ids = data_frame[\"Image_Label\"].apply(lambda x:x.split(\"_\")[0]).drop_duplicates().values\n      self.image_data_path = image_data_path / 'test_images_525'/ 'test_images_525'\n\n    if subset == 'train':\n      self.transforms = albumentations.Compose(self.training_augmentations() +                         self.validation_augmentations()\n                       )\n    else:\n      self.transforms = albumentations.Compose(self.validation_augmentations())\n\n  def fetch_masks(self,\n                  image_name: str):\n    \"\"\"\n    Create mask based on df, image name and shape.\n    \"\"\"\n    masks = numpy.zeros(self.response_shape, dtype=numpy.float32)\n    df = self.data_frame[self.data_frame[\"im_id\"] == image_name]\n\n    for idx, im_name in enumerate(df[\"im_id\"].values):\n      for classidx, classid in enumerate(self.categories.values()):\n        mpath = str(self.base_image_data / 'train_masks_525' / 'train_masks_525' / f'{classid}{im_name}')\n        mask = cv2.imread(mpath,\n                          cv2.IMREAD_GRAYSCALE)\n        if mask is None:\n          continue\n        mask = resize_image_cv(mask, self.image_size_T)\n        masks[:, :, classidx] = mask\n\n    masks = masks / 255.0\n    return masks\n\n  @staticmethod\n  def no_info_mask(img):\n    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)\n    lower = numpy.array([0, 0, 0], numpy.uint8)\n    upper = numpy.array([180, 255, 10], numpy.uint8)\n    return (~ (cv2.inRange(hsv, lower, upper) > 250)).astype(numpy.uint8)\n\n  def __getitem__(self, idx):\n    image_name = self.img_ids[idx]\n    img = cv2.imread(str(self.image_data_path / image_name))\n    img = resize_image_cv(img, self.image_size_T)\n    img_o = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n    if self.subset == 'test':\n      img_o = draugr.uint_hwc_to_chw_float(img_o)\n      return img_o, self.no_info_mask(img)\n\n    masks = self.fetch_masks(image_name)\n    if self.transforms:\n      augmented = self.transforms(image=img_o, mask=masks)\n      img_o = augmented[\"image\"]\n      masks = augmented[\"mask\"]\n    img_o = draugr.uint_hwc_to_chw_float(img_o)\n    masks = draugr.hwc_to_chw(masks)\n    return img_o, masks\n\n  def __len__(self):\n    return len(self.img_ids)\n\n  @staticmethod\n  def visualise(image,\n                mask,\n                original_image=None,\n                original_mask=None):\n    \"\"\"\n    Plot image and masks.\n    If two pairs of images and masks are passes, show both.\n    \"\"\"\n    fontsize = 14\n\n    if original_image is None and original_mask is None:\n      f, ax = pyplot.subplots(1, 5, figsize=(24, 24))\n\n      ax[0].imshow(image)\n      for i in range(4):\n        ax[i + 1].imshow(mask[:, :, i])\n        ax[i + 1].set_title(f\"Mask {CloudSegmentationDataset.categories[i]}\",\n                            fontsize=fontsize)\n    else:\n      f, ax = pyplot.subplots(2, 5, figsize=(24, 12))\n\n      ax[0, 0].imshow(original_image)\n      ax[0, 0].set_title(\"Original image\",\n                         fontsize=fontsize)\n\n      for i in range(4):\n        ax[0, i + 1].imshow(original_mask[:, :, i], vmin=0, vmax=1)\n        ax[0, i + 1].set_title(f\"Original mask {CloudSegmentationDataset.categories[i]}\",\n                               fontsize=fontsize)\n\n      ax[1, 0].imshow(image)\n      ax[1, 0].set_title(\"Transformed image\",\n                         fontsize=fontsize)\n\n      for i in range(4):\n        ax[1, i + 1].imshow(mask[:, :, i], vmin=0, vmax=1)\n        ax[1, i + 1].set_title(f\"Transformed mask {CloudSegmentationDataset.categories[i]}\",\n                               fontsize=fontsize)\n\n    pyplot.show()\n\n  @staticmethod\n  def visualise_prediction(\n    processed_image,\n    processed_mask,\n    original_image=None,\n    original_mask=None,\n    raw_image=None,\n    raw_mask=None\n    ):\n    \"\"\"\n    Plot image and masks.\n    If two pairs of images and masks are passes, show both.\n    \"\"\"\n    fontsize = 14\n\n    f, ax = pyplot.subplots(3, 5, figsize=(24, 12))\n\n    ax[0, 0].imshow(original_image)\n    ax[0, 0].set_title(\"Original image\",\n                       fontsize=fontsize)\n\n    for i in range(4):\n      ax[0, i + 1].imshow(original_mask[:, :, i], vmin=0, vmax=1)\n      ax[0, i + 1].set_title(f\"Original mask {CloudSegmentationDataset.categories[i]}\",\n                             fontsize=fontsize)\n\n    ax[1, 0].imshow(raw_image)\n    ax[1, 0].set_title(\"Raw image\", fontsize=fontsize)\n\n    for i in range(4):\n      ax[1, i + 1].imshow(raw_mask[:, :, i], vmin=0, vmax=1)\n      ax[1, i + 1].set_title(f\"Predicted mask {CloudSegmentationDataset.categories[i]}\",\n                             fontsize=fontsize)\n\n    ax[2, 0].imshow(processed_image)\n    ax[2, 0].set_title(\"Transformed image\",\n                       fontsize=fontsize)\n\n    for i in range(4):\n      ax[2, i + 1].imshow(processed_mask[:, :, i])\n      ax[2, i + 1].set_title(f\"Predicted mask with processing {CloudSegmentationDataset.categories[i]}\",\n                             fontsize=fontsize\n                             )\n\n    pyplot.show()\n\n  def plot_training_sample(self):\n    \"\"\"\n    Wrapper for `visualize` function.\n    \"\"\"\n    orig_transforms = self.transforms\n    self.transforms = None\n    image, mask = self.__getitem__(numpy.random.randint(0, self.__len__()))\n    print(image.shape)\n    print(mask.shape)\n    self.transforms = orig_transforms\n    image = draugr.float_chw_to_hwc_uint(image)\n    mask = draugr.chw_to_hwc(mask)\n    print(image.shape)\n    print(mask.shape)\n    augmented = orig_transforms(image=image, mask=mask)\n    augmented_image = augmented[\"image\"]\n    augmented_mask = augmented[\"mask\"]\n    print(augmented_image.shape)\n    print(augmented_mask.shape)\n    self.visualise(augmented_image,\n                   augmented_mask,\n                   original_image=image,\n                   original_mask=mask)\n\nif True:\n    ds = CloudSegmentationDataset(base_dataset_path, image_path)\n    ds.plot_training_sample()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model,\n                train_loader,\n                valid_loader,\n                criterion,\n                optimizer,\n                scheduler,\n                save_model_path: Path):\n  valid_loss_min = numpy.Inf  # track change in validation loss\n  E = tqdm(range(1, n_epochs + 1))\n  for epoch in E:\n    train_loss = 0.0\n    valid_loss = 0.0\n    dice_score = 0.0\n\n    model.train()\n    train_set = tqdm(train_loader, postfix={\"train_loss\":0.0})\n    for data, target in train_set:\n      data, target = data.to(global_torch_device(),dtype=torch.float), target.to(global_torch_device(),dtype=torch.float)\n      optimizer.zero_grad()\n      output, *_ = model(data)\n      output = torch.sigmoid(output)\n      loss = criterion(output, target)\n      loss.backward()\n      optimizer.step()\n      train_loss += loss.item() * data.size(0)\n      train_set.set_postfix(ordered_dict={\"train_loss\":loss.item()})\n\n    model.eval()\n    with torch.no_grad():\n      validation_set = tqdm(valid_loader, postfix={\"valid_loss\":0.0, \"dice_score\":0.0})\n      for data, target in validation_set:\n        data, target = data.to(global_torch_device(),dtype=torch.float), target.to(global_torch_device(),dtype=torch.float)\n\n        output, *_ = model(data)  # forward pass: compute predicted outputs by passing inputs to the model\n        output = torch.sigmoid(output)\n\n        loss = criterion(output, target)  # calculate the batch loss\n\n        valid_loss += loss.item() * data.size(0)  # update average validation loss\n        dice_cof = bool_dice(output.cpu().detach().numpy(), target.cpu().detach().numpy())\n        dice_score += dice_cof * data.size(0)\n        validation_set.set_postfix(ordered_dict={\"valid_loss\":loss.item(), \"dice_score\":dice_cof})\n\n    # calculate average losses\n    train_loss = train_loss / len(train_loader.dataset)\n    valid_loss = valid_loss / len(valid_loader.dataset)\n    dice_score = dice_score / len(valid_loader.dataset)\n\n    # print training/validation statistics\n    E.set_description(f'Epoch: {epoch}'\n                      f' Training Loss: {train_loss:.6f} '\n                      f'Validation Loss: {valid_loss:.6f} '\n                      f'Dice Score: {dice_score:.6f}')\n\n    # save model if validation loss has decreased\n    if valid_loss <= valid_loss_min:\n      print(f'Validation loss decreased ({valid_loss_min:.6f} --> {valid_loss:.6f}).  Saving model ...')\n      torch.save(model.state_dict(), str(save_model_path))\n      valid_loss_min = valid_loss\n\n    scheduler.step()\n\n  return model\n\ntrain_loader = DataLoader(CloudSegmentationDataset(base_dataset_path,\n                                                     image_path,\n                                                     subset=\"train\",\n                                                     ),\n                            batch_size=batch_size,\n                            shuffle=True,\n                            num_workers=num_workers\n                            )\nvalid_loader = DataLoader(CloudSegmentationDataset(base_dataset_path,\n                                                     image_path,\n                                                     subset=\"valid\",\n                                                     ),\n                            batch_size=batch_size,\n                            shuffle=False,\n                            num_workers=num_workers\n                            )\n\nmodel = SkipHourglassFissionNet(CloudSegmentationDataset.predictor_channels,\n                                  (CloudSegmentationDataset.response_channels,),\n                                  encoding_depth=encoding_depth)\nmodel.to(global_torch_device())\n\nif save_model_path.exists():\n    model.load_state_dict(torch.load(str(save_model_path)))  # load last model\n    print('loading previous model')\n\n  \noptimizer = torch.optim.SGD(model.parameters(), lr=lr)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer,\n                                                                   7,\n                                                                   eta_min=lr / 100,\n                                                                   last_epoch=-1)\n\n  \nmodel = train_model(model,\n                        train_loader,\n                        valid_loader,\n                        criterion,\n                        optimizer,\n                        scheduler,\n                        save_model_path)\n\n\nif save_model_path.exists():\n  model.load_state_dict(torch.load(str(save_model_path)))  # load best model\nmodel.eval()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Post Processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def post_process_minsize(mask, min_size):\n  \"\"\"\n  Post processing of each predicted mask, components with lesser number of pixels\n  than `min_size` are ignored\n  \"\"\"\n  num_component, component = cv2.connectedComponents(mask.astype(numpy.uint8))\n  predictions, num = numpy.zeros(mask.shape), 0\n  for c in range(1, num_component):\n    p = (component == c)\n    if p.sum() > min_size:\n      predictions[p] = 1\n      num += 1\n  return predictions\n\n\ndef threshold_mask(probability, threshold, min_size, psize):\n  \"\"\"\n  This is slightly different from other kernels as we draw convex hull here itself.\n  Post processing of each predicted mask, components with lesser number of pixels\n  than `min_size` are ignored\n  \"\"\"\n  mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]\n  mask = draw_convex_hull(mask.astype(numpy.uint8))\n  num_component, component = cv2.connectedComponents(mask.astype(numpy.uint8))\n  predictions = numpy.zeros(psize, numpy.float32)\n  num = 0\n  for c in range(1, num_component):\n    p = component == c\n    if p.sum() > min_size:\n      predictions[p] = 1\n      num += 1\n  return predictions, num\n\ndef threshold_grid_search(model, valid_loader, max_samples=threshold_samples):\n  ''' Grid Search for best Threshold '''\n\n  valid_masks = []\n  count = 0\n  tr = min(valid_loader.dataset.__len__(), max_samples)\n  probabilities = numpy.zeros((tr,\n                               *CloudSegmentationDataset.image_size_T),\n                              dtype=numpy.float32)\n  for data, targets in tqdm(valid_loader):\n    data = data.to(global_torch_device(),dtype=torch.float)\n    predictions, *_ = model(data)\n    predictions = torch.sigmoid(predictions)\n    predictions = predictions.cpu().detach().numpy()\n    targets = targets.cpu().detach().numpy()\n    for p in range(data.shape[0]):\n      pred, target = predictions[p], targets[p]\n      for mask_ in target:\n        valid_masks.append(mask_)\n      for probability in pred:\n        probabilities[count, :, :] = probability\n        count += 1\n      if count >= tr - 1:\n        break\n    if count >= tr - 1:\n      break\n\n  class_params = {}\n\n  for class_id in CloudSegmentationDataset.categories.keys():\n    print(CloudSegmentationDataset.categories[class_id])\n    attempts = []\n    for t in range(0, 100, 5):\n      t /= 100\n      print(t)\n      for ms in MIN_SIZES:\n        print(ms)\n        masks, d = [], []\n        for i in range(class_id, len(probabilities), 4):\n          probability_ = probabilities[i]\n          predict, num_predict = threshold_mask(probability_, t, ms,CloudSegmentationDataset.image_size_T)\n          masks.append(predict)\n        for i, j in zip(masks, valid_masks[class_id::4]):\n          if (i.sum() == 0) & (j.sum() == 0):\n            d.append(1)\n          else:\n            d.append(bool_dice(i, j))\n        attempts.append((t, ms, numpy.mean(d)))\n\n    attempts_df = pandas.DataFrame(attempts, columns=['threshold', 'size', 'dice'])\n    attempts_df = attempts_df.sort_values('dice', ascending=False)\n    print(attempts_df.head())\n    best_threshold = attempts_df['threshold'].values[0]\n    best_size = attempts_df['size'].values[0]\n    class_params[class_id] = (best_threshold, best_size)\n\n  return class_params\n\n\nclass_parameters = threshold_grid_search(model, valid_loader)\n\nfor _, (data, target) in zip(range(4),valid_loader):\n    data = data.to(global_torch_device(),dtype=torch.float)\n    output, *_ = model(data)\n    output = torch.sigmoid(output)\n    output= output[0].cpu().detach().numpy()\n    image_vis = data[0].cpu().detach().numpy()\n    mask = target[0].cpu().detach().numpy()\n\n    mask = chw_to_hwc(mask)\n    output = chw_to_hwc(output)\n    image_vis = float_chw_to_hwc_uint(image_vis)\n\n    pr_mask = numpy.zeros(CloudSegmentationDataset.response_shape)\n    for j in range(len(CloudSegmentationDataset.categories)):\n      probability_ = output[:, :, j]\n      thr, min_size = class_parameters[j][0], class_parameters[j][1]\n      pr_mask[:, :, j], _ = threshold_mask(probability_, thr, min_size, CloudSegmentationDataset.image_size_T)\n    CloudSegmentationDataset.visualise_prediction(image_vis,\n                                                  pr_mask,\n                                                  original_image=image_vis,\n                                                  original_mask=mask,\n                                                  raw_image=image_vis,\n                                                  raw_mask=output)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_submission(model, class_params, test_loader, submission_file_path = 'submission.csv'):\n  #encoded_pixels = []\n  submission_i = 0\n  number_of_pixels_saved = 0\n  df:pandas.DataFrame = test_loader.dataset.data_frame\n  a = df['Image_Label']\n\n  with open(submission_file_path, mode='w') as f:\n    f.write(\"Image_Label,EncodedPixels\\n\")\n    for data, black_mask in tqdm(test_loader):\n      data = data.to(global_torch_device(),dtype=torch.float)\n      output, *_ = model(data)\n      del data\n      output = torch.sigmoid(output)\n      output = output.cpu().detach().numpy()\n      black_masks = black_mask.cpu().detach().numpy()\n      for instance_i,black_mask in zip(output,black_masks):\n        for probability in instance_i:\n          thr, min_size = class_params[submission_i % 4][0], class_params[submission_i % 4][1]\n          black_mask = resize_image_cv(black_mask, final_mask_size_T)\n          probability = resize_image_cv(probability, final_mask_size_T)\n          predict, num_predict = threshold_mask(probability, thr, min_size, final_mask_size_T)\n          if num_predict == 0:\n            rle=''\n            #encoded_pixels.append('')\n          else:\n            number_of_pixels_saved += numpy.sum(predict)\n            predict_masked2 = numpy.multiply(predict, black_mask)\n            number_of_pixels_saved -= numpy.sum(predict_masked2)\n            rle = mask_to_run_length(predict_masked2)\n            #encoded_pixels.append(rle)\n\n          f.write(f\"{a[submission_i]},{rle}\\n\")\n          submission_i += 1\n\n    #df['EncodedPixels'] = encoded_pixels\n    #df.to_csv(submission_file_path, columns=['Image_Label', 'EncodedPixels'], index=False)\n\n  print(f\"Number of pixel saved {number_of_pixels_saved}\")\n\ntest_loader = DataLoader(CloudSegmentationDataset(base_dataset_path,\n                                                    image_path,\n                                                    subset='test'),\n                           batch_size=batch_size,\n                           shuffle=False,\n                           num_workers=num_workers)\n\nprepare_submission(model,\n                 class_parameters,\n                 test_loader\n                 )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}