{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Download HPA HE stained full-tissue sections\n\n- The [HPA Dictionary](https://www.proteinatlas.org/learn/dictionary) contains, among other things, normal tissue histology based on full tissue sections stained with hematoxylin-eosin (HE). \n- The images seem not to be available as TIF files to download, but as tiled JPGs at different magnifications as discovered by [hengck23](https://www.kaggle.com/hengck23) in [this](https://www.kaggle.com/competitions/hubmap-organ-segmentation/discussion/337692#1863834) discussion.\n- In the following notebook, I demonstrate how to download the tiled JPGs at different magnifications. ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Get filename(s)\n\n1. Go to https://www.proteinatlas.org/learn/dictionary and select any organ you want, e.g. the Kidney.\n2. Once on the organ-specific page that shows the tissue image, open the developer tools and go to the 'Network' tab.\n3. Refresh the page and find the request for one of the JPGs, e.g. '0_0.jpg'and copy the value of the request url, e.g. https://images.proteinatlas.org/dictionary_images/fileup5e998cc3050ef333190901_files/8/0_0.jpg\n4. Copy the filename part of the url ('fileup5f9fe9fbd8ca4499343163_files') and paste it into the FILE_NAMES dict variable below\n5. Repeat steps for other organs, e.g. 'fileup5e998cc3050ef333190901_files' for lung","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import time\nfrom itertools import product\nfrom pathlib import Path\nfrom typing import Dict\n\nimport requests\nfrom matplotlib import pyplot as plt\nfrom io import BytesIO\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:39:57.684664Z","iopub.execute_input":"2022-07-22T15:39:57.685141Z","iopub.status.idle":"2022-07-22T15:39:57.692876Z","shell.execute_reply.started":"2022-07-22T15:39:57.685109Z","shell.execute_reply":"2022-07-22T15:39:57.690826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Settings","metadata":{}},{"cell_type":"code","source":"BASE_LEVEL = 8 # 8 seems to be the base magnification level. At level 8 there is only 0_0.jpg. \n\nOUTPUT_DIR = Path(\".\") / \"dictionary_images\"\nOUTPUT_DIR.mkdir(exist_ok=True)\n\nBASE_URL = \"https://images.proteinatlas.org/dictionary_images\"\n\nFILE_NAMES = {\n    \"kidney\": \"fileup5f9fe9fbd8ca4499343163_files\",\n    \"lung\": \"fileup5e998cc3050ef333190901_files\",\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:39:57.709907Z","iopub.execute_input":"2022-07-22T15:39:57.710415Z","iopub.status.idle":"2022-07-22T15:39:57.717843Z","shell.execute_reply.started":"2022-07-22T15:39:57.710367Z","shell.execute_reply":"2022-07-22T15:39:57.716825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"code","source":"def download_images(\n    magnification_level: int,\n    base_level: int = BASE_LEVEL,\n    data_dir: Path = OUTPUT_DIR,\n    base_url: str = BASE_URL,\n    file_names: Dict[str, str] = FILE_NAMES,\n):\n    assert magnification_level >= base_level, \"Magnification level needs to be >= base level\"\n    \n    for organ, file_name in file_names.items():\n        save_dir = data_dir / f\"{organ}_{file_name}\"\n        save_dir.mkdir(exist_ok=True)\n        print(save_dir)\n\n        nrows = 1\n        image_found = True\n        while image_found:\n            image_names = (\n                f\"{comb[0]}_{comb[1]}.jpg\"\n                for comb in list(set(product(range(nrows), repeat=2)) - set(product(range(nrows - 1), repeat=2)))\n            )\n\n            num_images_found = 0\n            for image_name in image_names:\n                url = f\"{base_url}/{file_name}/{magnification_level}/{image_name}\"\n\n                response = requests.get(url, timeout=1)\n                time.sleep(0.1)\n                if response.status_code == 200:\n                    image = Image.open(BytesIO(response.content))\n                    image.save(save_dir / f\"{magnification_level}_{image_name}\")\n\n                    print(f\"Found {magnification_level}_{image_name} with size {image.size}\")\n                    num_images_found += 1\n\n            image_found = num_images_found > 0\n            if image_found:\n                nrows += 1","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:39:57.720219Z","iopub.execute_input":"2022-07-22T15:39:57.720781Z","iopub.status.idle":"2022-07-22T15:39:57.737369Z","shell.execute_reply.started":"2022-07-22T15:39:57.720714Z","shell.execute_reply":"2022-07-22T15:39:57.735857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_tiles(\n    magnification_level: int,\n    tile_path: str = \"dictionary_images/kidney_fileup5e998cc3050ef333190901_files\",\n):\n    image_paths = sorted(Path(tile_path).glob(f\"{magnification_level}_*.jpg\"))\n    image_names = [Path(image_path).name.split(\".\")[0] for image_path in image_paths]\n\n    col_idxs = [int(image_name.split(\"_\")[1]) for image_name in image_names]\n    row_idxs = [int(image_name.split(\"_\")[2]) for image_name in image_names]\n\n    ncols = max(col_idxs) + 1\n    nrows = max(row_idxs) + 1\n    fig, _ = plt.subplots(figsize=(3 * nrows, 3 * ncols))\n\n    for idx, image_path in enumerate(image_paths):\n        img_name = Path(image_path).name.split(\".\")[0]\n        col_idx = int(img_name.split(\"_\")[1])\n        row_idx = int(img_name.split(\"_\")[2])\n        plt.subplot(nrows, ncols, col_idx + row_idx * ncols + 1)\n\n        image = plt.imread(image_path)\n\n        plt.title(img_name)\n        plt.imshow(image)\n        plt.tight_layout()\n        plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:44:22.262225Z","iopub.execute_input":"2022-07-22T15:44:22.262681Z","iopub.status.idle":"2022-07-22T15:44:22.276827Z","shell.execute_reply.started":"2022-07-22T15:44:22.262644Z","shell.execute_reply":"2022-07-22T15:44:22.275832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Magnification Level 8 (Base Level)","metadata":{}},{"cell_type":"code","source":"magnification_level = 8\n\ndownload_images(magnification_level)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:39:57.753011Z","iopub.execute_input":"2022-07-22T15:39:57.754016Z","iopub.status.idle":"2022-07-22T15:40:04.334243Z","shell.execute_reply.started":"2022-07-22T15:39:57.753962Z","shell.execute_reply":"2022-07-22T15:40:04.332209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tiles(magnification_level, tile_path=\"dictionary_images/kidney_fileup5f9fe9fbd8ca4499343163_files\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:40:04.336109Z","iopub.execute_input":"2022-07-22T15:40:04.336491Z","iopub.status.idle":"2022-07-22T15:40:04.498821Z","shell.execute_reply.started":"2022-07-22T15:40:04.336459Z","shell.execute_reply":"2022-07-22T15:40:04.497335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tiles(magnification_level, tile_path=\"dictionary_images/lung_fileup5e998cc3050ef333190901_files\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:40:04.500773Z","iopub.execute_input":"2022-07-22T15:40:04.502341Z","iopub.status.idle":"2022-07-22T15:40:04.603192Z","shell.execute_reply.started":"2022-07-22T15:40:04.502275Z","shell.execute_reply":"2022-07-22T15:40:04.602282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Magnification Level 9","metadata":{}},{"cell_type":"code","source":"magnification_level = 9\n\ndownload_images(magnification_level)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:40:04.604337Z","iopub.execute_input":"2022-07-22T15:40:04.610376Z","iopub.status.idle":"2022-07-22T15:40:18.912430Z","shell.execute_reply.started":"2022-07-22T15:40:04.610290Z","shell.execute_reply":"2022-07-22T15:40:18.911387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tiles(magnification_level, tile_path=\"dictionary_images/kidney_fileup5f9fe9fbd8ca4499343163_files\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:40:18.914049Z","iopub.execute_input":"2022-07-22T15:40:18.914442Z","iopub.status.idle":"2022-07-22T15:40:19.327808Z","shell.execute_reply.started":"2022-07-22T15:40:18.914407Z","shell.execute_reply":"2022-07-22T15:40:19.326609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tiles(magnification_level, tile_path=\"dictionary_images/lung_fileup5e998cc3050ef333190901_files\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:40:19.331849Z","iopub.execute_input":"2022-07-22T15:40:19.332363Z","iopub.status.idle":"2022-07-22T15:40:19.751253Z","shell.execute_reply.started":"2022-07-22T15:40:19.332317Z","shell.execute_reply":"2022-07-22T15:40:19.749724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Magnification Level 10","metadata":{}},{"cell_type":"code","source":"magnification_level = 10\n\ndownload_images(magnification_level)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:40:19.752926Z","iopub.execute_input":"2022-07-22T15:40:19.753661Z","iopub.status.idle":"2022-07-22T15:40:45.487447Z","shell.execute_reply.started":"2022-07-22T15:40:19.753616Z","shell.execute_reply":"2022-07-22T15:40:45.486424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tiles(magnification_level, tile_path=\"dictionary_images/kidney_fileup5f9fe9fbd8ca4499343163_files\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:40:45.489094Z","iopub.execute_input":"2022-07-22T15:40:45.489469Z","iopub.status.idle":"2022-07-22T15:40:46.571469Z","shell.execute_reply.started":"2022-07-22T15:40:45.489435Z","shell.execute_reply":"2022-07-22T15:40:46.570053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tiles(magnification_level, tile_path=\"dictionary_images/lung_fileup5e998cc3050ef333190901_files\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:40:46.573239Z","iopub.execute_input":"2022-07-22T15:40:46.573582Z","iopub.status.idle":"2022-07-22T15:40:47.440198Z","shell.execute_reply.started":"2022-07-22T15:40:46.573552Z","shell.execute_reply":"2022-07-22T15:40:47.438830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Magnification Level 11","metadata":{}},{"cell_type":"code","source":"magnification_level = 11\n\ndownload_images(magnification_level)","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:40:47.442468Z","iopub.execute_input":"2022-07-22T15:40:47.443343Z","iopub.status.idle":"2022-07-22T15:42:09.290346Z","shell.execute_reply.started":"2022-07-22T15:40:47.443295Z","shell.execute_reply":"2022-07-22T15:42:09.288999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tiles(magnification_level, tile_path=\"dictionary_images/kidney_fileup5f9fe9fbd8ca4499343163_files\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:44:27.697608Z","iopub.execute_input":"2022-07-22T15:44:27.698199Z","iopub.status.idle":"2022-07-22T15:44:32.339432Z","shell.execute_reply.started":"2022-07-22T15:44:27.698154Z","shell.execute_reply":"2022-07-22T15:44:32.337537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tiles(magnification_level, tile_path=\"dictionary_images/lung_fileup5e998cc3050ef333190901_files\")","metadata":{"execution":{"iopub.status.busy":"2022-07-22T15:44:32.341964Z","iopub.execute_input":"2022-07-22T15:44:32.343097Z","iopub.status.idle":"2022-07-22T15:44:35.894007Z","shell.execute_reply.started":"2022-07-22T15:44:32.343044Z","shell.execute_reply":"2022-07-22T15:44:35.892717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}