{"cells":[{"metadata":{},"cell_type":"markdown","source":"### get torchvision utils for mask-rcnn"},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.append(\"../input/maskrcnn-utils/\")","execution_count":1,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### install latest torchvision"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!pip install -U torchvision","execution_count":2,"outputs":[{"output_type":"stream","text":"Collecting torchvision\n\u001b[?25l Downloading https://files.pythonhosted.org/packages/2e/45/0f2f3062c92d9cf1d5d7eabd3cae88cea9affbd2b17fb1c043627838cb0a/torchvision-0.3.0-cp36-cp36m-manylinux1_x86_64.whl (2.6MB)\n\u001b[K 100% |████████████████████████████████| 2.6MB 14.1MB/s ta 0:00:01\n\u001b[?25hRequirement already satisfied, skipping upgrade: pillow>=4.1.1 in /opt/conda/lib/python3.6/site-packages (from torchvision) (5.1.0)\nCollecting torch>=1.1.0 (from torchvision)\n\u001b[?25l Downloading https://files.pythonhosted.org/packages/69/60/f685fb2cfb3088736bafbc9bdbb455327bdc8906b606da9c9a81bae1c81e/torch-1.1.0-cp36-cp36m-manylinux1_x86_64.whl (676.9MB)\n\u001b[K 100% |████████████████████████████████| 676.9MB 58kB/s eta 0:00:01 11% |███▊ | 78.3MB 35.5MB/s eta 0:00:17 13% |████▍ | 92.8MB 17.1MB/s eta 0:00:35 15% |█████ | 106.9MB 31.0MB/s eta 0:00:19 22% |███████▏ | 152.0MB 36.8MB/s eta 0:00:15 26% |████████▍ | 178.4MB 34.7MB/s eta 0:00:15 33% |██████████▊ | 226.9MB 34.2MB/s eta 0:00:14 38% |████████████▏ | 258.2MB 17.3MB/s eta 0:00:25 39% |████████████▊ | 270.2MB 34.9MB/s eta 0:00:12 42% |█████████████▌ | 286.5MB 36.7MB/s eta 0:00:11 53% |█████████████████▏ | 362.6MB 33.4MB/s eta 0:00:10 61% |███████████████████▉ | 419.4MB 34.3MB/s eta 0:00:08 37.7MB/s eta 0:00:07 62% |████████████████████ | 422.3MB 22.8MB/s eta 0:00:12 62% |████████████████████ | 423.8MB 20.0MB/s eta 0:00:13 62% |████████████████████ | 424.7MB 20.5MB/s eta 0:00:13 62% |████████████████████▏ | 425.7MB 20.3MB/s eta 0:00:13�█████████████████▏ | 426.1MB 22.3MB/s eta 0:00:12 76% |████████████████████████▋ | 520.4MB 33.9MB/s eta 0:00:05 81% |██████████████████████████ | 548.6MB 34.5MB/s eta 0:00:04 84% |███████████████████████████ | 571.5MB 33.1MB/s eta 0:00:04��██████████████▋ | 583.9MB 5.3MB/s eta 0:00:18 89% |████████████████████████████▋ | 605.9MB 32.4MB/s eta 0:00:03 98% |███████████████████████████████▋| 668.5MB 35.4MB/s eta 0:00:01\n\u001b[?25hRequirement already satisfied, skipping upgrade: numpy in /opt/conda/lib/python3.6/site-packages (from torchvision) (1.16.3)\nRequirement already satisfied, skipping upgrade: six in /opt/conda/lib/python3.6/site-packages (from torchvision) (1.12.0)\nInstalling collected packages: torch, torchvision\n Found existing installation: torch 1.0.1.post2\n Uninstalling torch-1.0.1.post2:\n Successfully uninstalled torch-1.0.1.post2\n Found existing installation: torchvision 0.2.2\n Uninstalling torchvision-0.2.2:\n Successfully uninstalled torchvision-0.2.2\nSuccessfully installed torch-1.1.0 torchvision-0.3.0\n\u001b[33mYou are using pip version 19.0.3, however version 19.1.1 is available.\nYou should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"### import everything useful"},{"metadata":{"trusted":true},"cell_type":"code","source":"import collections\nimport os\nimport numpy as np\nimport torch\nimport torch.utils.data\nfrom PIL import Image, ImageFile\nimport pandas as pd\nfrom tqdm import tqdm\nfrom numba import jit\nfrom model import get_instance_segmentation_model\nimport torch\nfrom engine import train_one_epoch, evaluate\nimport utils\nimport transforms as T\nfrom tqdm import tqdm\nimport numpy as np\nimport torch\nimport pandas as pd\nfrom model import get_instance_segmentation_model\nfrom torchvision import transforms\nfrom PIL import Image\nimport itertools\n\nImageFile.LOAD_TRUNCATED_IMAGES = True","execution_count":5,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### create dataset class for getting batches of images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def rle_decode(mask_rle, shape):\n '''\n mask_rle: run-length as string formated: [start0] [length0] [start1] [length1]... in 1d array\n shape: (height,width) of array to return\n Returns numpy array according to the shape, 1 - mask, 0 - background\n '''\n shape = (shape[1], shape[0])\n s = mask_rle.split()\n # gets starts & lengths 1d arrays\n starts, lengths = [np.asarray(x, dtype=int) for x in (s[0::2], s[1::2])]\n starts -= 1\n # gets ends 1d array\n ends = starts + lengths\n # creates blank mask image 1d array\n img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n # sets mark pixles\n for lo, hi in zip(starts, ends):\n img[lo:hi] = 1\n # reshape as a 2d mask image\n return img.reshape(shape).T # Needed to align to RLE direction\n\n\n\nclass FashionDataset(torch.utils.data.Dataset):\n def __init__(self, image_dir, df_path, height, width, transforms=None):\n self.transforms = transforms\n self.image_dir = image_dir\n self.df = pd.read_csv(df_path, nrows=10000)\n self.height = height\n self.width = width\n self.image_info = collections.defaultdict(dict)\n self.df['CategoryId'] = self.df.ClassId.apply(lambda x: str(x).split(\"_\")[0])\n temp_df = self.df.groupby('ImageId')['EncodedPixels', 'CategoryId'].agg(lambda x: list(x)).reset_index()\n size_df = self.df.groupby('ImageId')['Height', 'Width'].mean().reset_index()\n temp_df = temp_df.merge(size_df, on='ImageId', how='left')\n for index, row in tqdm(temp_df.iterrows(), total=len(temp_df)):\n image_id = row['ImageId']\n image_path = os.path.join(self.image_dir, image_id)\n self.image_info[index][\"image_id\"] = image_id\n self.image_info[index][\"image_path\"] = image_path\n self.image_info[index][\"width\"] = self.width\n self.image_info[index][\"height\"] = self.height\n self.image_info[index][\"labels\"] = row[\"CategoryId\"]\n self.image_info[index][\"orig_height\"] = row[\"Height\"]\n self.image_info[index][\"orig_width\"] = row[\"Width\"]\n self.image_info[index][\"annotations\"] = row[\"EncodedPixels\"]\n\n def __getitem__(self, idx):\n # load images ad masks\n img_path = self.image_info[idx][\"image_path\"]\n img = Image.open(img_path).convert(\"RGB\")\n img = img.resize((self.width, self.height), resample=Image.BILINEAR)\n\n info = self.image_info[idx]\n mask = np.zeros((len(info['annotations']), self.width, self.height), dtype=np.uint8)\n labels = []\n for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):\n sub_mask = rle_decode(annotation, (info['orig_height'], info['orig_width']))\n sub_mask = Image.fromarray(sub_mask)\n sub_mask = sub_mask.resize((self.width, self.height), resample=Image.BILINEAR)\n mask[m, :, :] = sub_mask\n labels.append(int(label) + 1)\n\n num_objs = len(labels)\n boxes = []\n new_labels = []\n new_masks = []\n\n for i in range(num_objs):\n try:\n pos = np.where(mask[i, :, :])\n xmin = np.min(pos[1])\n xmax = np.max(pos[1])\n ymin = np.min(pos[0])\n ymax = np.max(pos[0])\n if abs(xmax - xmin) >= 20 and abs(ymax - ymin) >= 20:\n boxes.append([xmin, ymin, xmax, ymax])\n new_labels.append(labels[i])\n new_masks.append(mask[i, :, :])\n except ValueError:\n continue\n\n if len(new_labels) == 0:\n boxes.append([0, 0, 20, 20])\n new_labels.append(0)\n new_masks.append(mask[0, :, :])\n\n nmx = np.zeros((len(new_masks), self.width, self.height), dtype=np.uint8)\n for i, n in enumerate(new_masks):\n nmx[i, :, :] = n\n\n boxes = torch.as_tensor(boxes, dtype=torch.float32)\n labels = torch.as_tensor(new_labels, dtype=torch.int64)\n masks = torch.as_tensor(nmx, dtype=torch.uint8)\n\n image_id = torch.tensor([idx])\n area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n iscrowd = torch.zeros((num_objs,), dtype=torch.int64)\n\n target = {}\n target[\"boxes\"] = boxes\n target[\"labels\"] = labels\n target[\"masks\"] = masks\n target[\"image_id\"] = image_id\n target[\"area\"] = area\n target[\"iscrowd\"] = iscrowd\n\n if self.transforms is not None:\n img, target = self.transforms(img, target)\n\n return img, target\n\n def __len__(self):\n return len(self.image_info)","execution_count":6,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train the model"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"def get_transform(train):\n transforms = []\n # converts the image, a PIL image, into a PyTorch Tensor\n transforms.append(T.ToTensor())\n if train:\n # during training, randomly flip the training images\n # and ground-truth for data augmentation\n transforms.append(T.RandomHorizontalFlip(0.5))\n return T.Compose(transforms)\n\n\nnum_classes = 46 + 1\ndevice = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n\ndataset_train = FashionDataset(\"../input/imaterialist-fashion-2019-FGVC6/train/\",\n \"../input/imaterialist-fashion-2019-FGVC6/train.csv\",\n 256,\n 256,\n transforms=get_transform(train=True))\n\n\nmodel_ft = get_instance_segmentation_model(num_classes)\nmodel_ft.to(device)\n\ndata_loader = torch.utils.data.DataLoader(\n dataset_train, batch_size=4, shuffle=True, num_workers=8,\n collate_fn=lambda x: tuple(zip(*x)))\n\nparams = [p for p in model_ft.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=0.001,\n momentum=0.9, weight_decay=0.0005)\n\nlr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,\n step_size=5,\n gamma=0.1)\nnum_epochs = 10\n\nfor epoch in range(num_epochs):\n train_one_epoch(model_ft, optimizer, data_loader, device, epoch, print_freq=10)\n lr_scheduler.step()\n\ntorch.save(model_ft.state_dict(), \"model.bin\")","execution_count":7,"outputs":[{"output_type":"stream","text":"100%|██████████| 1355/1355 [00:00<00:00, 5810.63it/s]\n","name":"stderr"},{"output_type":"error","ename":"AttributeError","evalue":"module 'utils' has no attribute 'MetricLogger'","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m \u001b[0mtrain_one_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_ft\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprint_freq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 40\u001b[0m \u001b[0mlr_scheduler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/kaggle/input/maskrcnn-utils/engine.py\u001b[0m in \u001b[0;36mtrain_one_epoch\u001b[0;34m(model, optimizer, data_loader, device, epoch, print_freq)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtrain_one_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprint_freq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0mmetric_logger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMetricLogger\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdelimiter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\" \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 16\u001b[0m \u001b[0mmetric_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_meter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSmoothedValue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwindow_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfmt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'{value:.6f}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0mheader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Epoch: [{}]'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mAttributeError\u001b[0m: module 'utils' has no attribute 'MetricLogger'"]}]},{"metadata":{},"cell_type":"markdown","source":"### make predictions"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"def refine_masks(masks, labels):\n # Compute the areas of each mask\n areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)\n # Masks are ordered from smallest to largest\n mask_index = np.argsort(areas)\n # One reference mask is created to be incrementally populated\n union_mask = {k:np.zeros(masks.shape[:-1], dtype=bool) for k in np.unique(labels)}\n # Iterate from the smallest, so smallest ones are preserved\n for m in mask_index:\n label = labels[m]\n masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask[label]))\n union_mask[label] = np.logical_or(masks[:, :, m], union_mask[label])\n # Reorder masks\n refined = list()\n for m in range(masks.shape[-1]):\n mask = masks[:, :, m].ravel(order='F')\n rle = to_rle(mask)\n label = labels[m] - 1\n refined.append([masks[:, :, m], rle, label])\n return refined\n\n\nnum_classes = 46 + 1\n\ndataset_test = FashionDataset(\"../input/imaterialist-fashion-2019-FGVC6/test/\", \n \"../input/imaterialist-fashion-2019-FGVC6/sample_submission.csv\", 512, 512,\n transforms=None)\n\nsample_df = pd.read_csv(\"../input/imaterialist-fashion-2019-FGVC6/sample_submission.csv\")\n\n\nmodel_ft = get_instance_segmentation_model(num_classes)\nmodel_ft.load_state_dict(torch.load(\"model.bin\"))\nmodel_ft = model_ft.to(device)\n\nfor param in model_ft.parameters():\n param.requires_grad = False\n\nmodel_ft.eval()\n\n\nsub_list = []\nmissing_count = 0\nsubmission = []\nctr = 0\n\ntk0 = tqdm(range(3200))\ntt = transforms.ToTensor()\nfor i in tk0:\n img = dataset_test[i]\n img = tt(img)\n result = model_ft([img.to(device)])[0]\n masks = np.zeros((512, 512, len(result[\"masks\"])))\n for j, m in enumerate(result[\"masks\"]):\n res = transforms.ToPILImage()(result[\"masks\"][j].permute(1, 2, 0).cpu().numpy())\n res = np.asarray(res.resize((512, 512), resample=Image.BILINEAR))\n masks[:, :, j] = (res[:, :] * 255. > 127).astype(np.uint8)\n\n lbls = result['labels'].cpu().numpy()\n scores = result['scores'].cpu().numpy()\n\n best_idx = 0\n for scr in scores:\n if scr > 0.8:\n best_idx += 1\n\n if best_idx == 0:\n sub_list.append([sample_df.loc[i, 'ImageId'], '1 1', 23])\n missing_count += 1\n continue\n\n if masks.shape[-1] > 0:\n #lll = mask_to_rle(masks[:, :, :4], scores[:4], lbls[:4])\n masks = refine_masks(masks[:, :, :best_idx], lbls[:best_idx])\n for m, rle, label in masks:\n sub_list.append([sample_df.loc[i, 'ImageId'], ' '.join(list(map(str, list(rle)))), label])\n else:\n sub_list.append([sample_df.loc[i, 'ImageId'], '1 1', 23])\n missing_count += 1\n\nsubmission_df = pd.DataFrame(sub_list, columns=sample_df.columns.values)\nprint(\"Total image results: \", submission_df['ImageId'].nunique())\nprint(\"Missing Images: \", missing_count)\nsubmission_df = submission_df[submission_df.EncodedPixels.notnull()]\nfor row in range(len(submission_df)):\n line = submission_df.iloc[row,:]\n submission_df.iloc[row, 1] = line['EncodedPixels'].replace('.0','')\nsubmission_df.head()\nsubmission_df.to_csv(\"submission.csv\", index=False)","execution_count":8,"outputs":[{"output_type":"error","ename":"KeyError","evalue":"\"Columns not found: 'Width', 'Height'\"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)","\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 25\u001b[0m dataset_test = FashionDataset(\"../input/imaterialist-fashion-2019-FGVC6/test/\", \n\u001b[1;32m 26\u001b[0m \u001b[0;34m\"../input/imaterialist-fashion-2019-FGVC6/sample_submission.csv\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m512\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m512\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m transforms=None)\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0msample_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"../input/imaterialist-fashion-2019-FGVC6/sample_submission.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, image_dir, df_path, height, width, transforms)\u001b[0m\n\u001b[1;32m 32\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'CategoryId'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mClassId\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0mtemp_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ImageId'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'EncodedPixels'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'CategoryId'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m \u001b[0msize_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ImageId'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Height'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Width'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 35\u001b[0m \u001b[0mtemp_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtemp_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmerge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ImageId'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'left'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 36\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrow\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterrows\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_df\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.6/site-packages/pandas/core/base.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 255\u001b[0m \u001b[0mbad_keys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdifference\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 256\u001b[0m raise KeyError(\"Columns not found: {missing}\"\n\u001b[0;32m--> 257\u001b[0;31m .format(missing=str(bad_keys)[1:-1]))\n\u001b[0m\u001b[1;32m 258\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gotitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mndim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 259\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyError\u001b[0m: \"Columns not found: 'Width', 'Height'\""]}]},{"metadata":{},"cell_type":"markdown","source":"### result\n\nTo run the code properly, download the attached dataset and run it locally instead of kaggle kernels. :) \n\nThe approach wont give you 0.17+ directly. It requires very small modifications to get that kind of score. I’ll leave those modifcations as an exercise to the reader.\n\nIf you have any questions, feel free to ask."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}