{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"!pip install albumentations -qqq --upgrade","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom matplotlib import pyplot as plt\nimport albumentations as A\nfrom tqdm.notebook import tqdm\nimport torch","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install wandb -qqq --upgrade\nimport wandb\nwandb.login()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# os.environ[\"KAGGLE_USERNAME\"] = \"vincenttu\"\n# os.environ[\"KAGGLE_KEY\"] = \"\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Original data.\n# %cd input/\n# !kaggle competitions download -c hubmap-organ-segmentation\n# !unzip -q hubmap-organ-segmentation.zip\n# !sudo rm hubmap-organ-segmentation.zip\n# %cd ../\n\n# 256x256 extracted imgs and masks.\n# %cd input/mmsegmentation_256x256/\n# !kaggle datasets download -d w3579628328/mmsegmentation256x256\n# !unzip -q mmsegmentation256x256.zip\n# !sudo rm mmsegmentation256x256.zip\n# % cd../../","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv(\"./input/train.csv\")\ntest_csv = pd.read_csv(\"./input/test.csv\")\nsample_submission = pd.read_csv(\"./input/sample_submission.csv\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset\nfrom albumentations.pytorch.transforms import ToTensorV2\n\nclass TrainDataset(Dataset):\n    def __init__(self, paths, transforms=None, img_size=(384, 384), preserve_aspect=True):\n        self.paths = paths\n        self.transforms = transforms\n        self.img_size = img_size\n        self.preserve_aspect = preserve_aspect\n        \n        if self.preserve_aspect:\n            self.start_transforms = A.Compose([\n                A.LongestMaxSize(self.img_size[0]),  # We always resize to square.\n                A.PadIfNeeded(*self.img_size)\n            ])\n        else:\n            self.start_transforms = A.Compose([\n                A.Resize(*self.img_size),\n            ])\n        \n        self.end_transforms = A.Compose([\n          ToTensorV2()\n        ])\n        \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, idx):\n        p = self.paths[idx]\n        img_p, mask_p = p[0], p[1]\n        \n        img = cv2.imread(img_p, cv2.COLOR_BGR2RGB)\n        mask = cv2.imread(mask_p, cv2.IMREAD_GRAYSCALE)\n        \n        # 0 is for empty. \n        # 1 -> kidney\n        # 2 -> prostate\n        # 3 -> large intestine\n        # 4 -> spleen\n        # 5 -> lung\n        # This info might be useful later. For now, our dataset will be agnostic to the type of organ.\n        mask = np.where(mask > 0, 1, 0)\n        \n        # Resize (preserve aspect ratio with padding, optional).\n        augmented = self.start_transforms(image=img, mask=mask)\n        img, mask = augmented['image'], augmented['mask']\n        \n        # Apply augmentations.\n        if self.transforms is not None:\n            augmented = self.transforms(image=img, mask=mask)\n            img, mask = augmented['image'], augmented['mask']\n        \n        # Convert to torch tensors.\n        img = self.end_transforms(image=img)[\"image\"]\n        mask = self.end_transforms(image=mask)[\"image\"]\n        \n        return {\n            \"images\": img,\n            \"labels\": mask\n        }","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths = [(img_p, img_p.replace(\"train\", \"masks\")) for img_p in glob(\"./input/mmsegmentation_256x256/train/*\")]\nds = TrainDataset(paths, \n                  transforms=None, \n                  img_size=(256, 256), \n                  preserve_aspect=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1 -> kidney <br>\n2 -> prostate <br>\n3 -> large intestine <br>\n4 -> spleen <br>\n5 -> lung","metadata":{}},{"cell_type":"markdown","source":"# Logging Visuals onto Wandb","metadata":{}},{"cell_type":"code","source":"run = wandb.init(project=\"HuBMAP_HPA_dataviz\", name=\"viz_dataset\", dir=\"/tmp\")\n\ndata = []\nfor i in tqdm(range(len(ds))):\n    patient_id = int(ds.paths[i][0].split(\"/\")[-1].split(\".\")[0].split(\"_\")[0])\n    organ = train_csv[train_csv.id == patient_id][\"organ\"].values[0]\n    age = train_csv[train_csv.id == patient_id][\"age\"].values[0]\n    sex = train_csv[train_csv.id == patient_id][\"sex\"].values[0]\n\n    img, label = ds[i][\"images\"], ds[i][\"labels\"]\n    data.append([patient_id, organ, age, sex, wandb.Image(img.permute(1, 2, 0).numpy()), wandb.Image(label.permute(1, 2, 0).numpy())])\n\nmy_table = wandb.Table(data=data, \n                       columns=[\"id\", \"organ\", \"age\", \"sex\", \"img\", \"label\"])\n\nrun.log({\n    \"my_table\": my_table\n})\n\nrun.finish()","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Take a look at the results: https://wandb.ai/vincenttu/HuBMAP_HPA_dataviz?workspace=user-vincenttu!","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}