{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport glob\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom ipywidgets import interact\nfrom PIL import Image\nfrom math import ceil\nfrom bs4 import BeautifulSoup\nimport requests\nfrom io import StringIO, BytesIO\nfrom tqdm.auto import tqdm\n\n!mkdir -p images","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:19:23.004207Z","iopub.execute_input":"2022-07-19T11:19:23.004576Z","iopub.status.idle":"2022-07-19T11:19:23.791476Z","shell.execute_reply.started":"2022-07-19T11:19:23.004545Z","shell.execute_reply":"2022-07-19T11:19:23.790289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r = requests.get(\"https://www.proteinatlas.org/api/search_download.php?search=&columns=g&compress=no&format=tsv\")\nstring = StringIO(r.text)\ndf = pd.read_csv(string, sep=\"\\t\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:17:33.368067Z","iopub.execute_input":"2022-07-19T11:17:33.368914Z","iopub.status.idle":"2022-07-19T11:17:42.412790Z","shell.execute_reply.started":"2022-07-19T11:17:33.368853Z","shell.execute_reply":"2022-07-19T11:17:42.411702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"organs = ['prostate']#, 'spleen', 'kidney', 'colon']\n\nmetadata = []\n\npbar = tqdm(df.index[:200])\n\nimg_idx = 0\nfor idx in pbar:\n    continue\n    ens = df.loc[idx, \"Ensembl\"]\n    gene = df.loc[idx, \"Gene\"]\n    for organ in organs:\n        url = f\"https://www.proteinatlas.org/{ens}-{gene}/tissue/{organ}\"\n        \n        r = requests.get(url)\n        images = BeautifulSoup(r.text, 'html.parser').findAll('img')\n        links = ['https:' + img['src'].replace('_medium', '') for img in images if img['src'].startswith('//images.proteinatlas.org')]\n        \n        for link in links:\n#             img_name = link.split('/')[-1]\n            img_name = f\"{organ}_{img_idx}.jpg\"\n            img_idx += 1\n            r = requests.get(link)\n            img = Image.open(BytesIO(r.content))\n            img.save(os.path.join(\"./images/\" + img_name))\n            metadata.append([img_name, organ])\n            \n            pbar.set_description(f\"{len(metadata):>5d} imgs saved\")\n            pbar.refresh()","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:20:52.613480Z","iopub.execute_input":"2022-07-19T11:20:52.613903Z","iopub.status.idle":"2022-07-19T11:29:26.864956Z","shell.execute_reply.started":"2022-07-19T11:20:52.613869Z","shell.execute_reply":"2022-07-19T11:29:26.863861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(metadata).to_csv(\"./hpa_unlabeled.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-19T11:31:28.146693Z","iopub.execute_input":"2022-07-19T11:31:28.147206Z","iopub.status.idle":"2022-07-19T11:31:28.156735Z","shell.execute_reply.started":"2022-07-19T11:31:28.147167Z","shell.execute_reply":"2022-07-19T11:31:28.155917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip images.zip -q -r -9 images","metadata":{"execution":{"iopub.status.busy":"2022-07-19T10:36:56.265858Z","iopub.execute_input":"2022-07-19T10:36:56.266336Z","iopub.status.idle":"2022-07-19T10:39:18.879400Z","shell.execute_reply.started":"2022-07-19T10:36:56.266296Z","shell.execute_reply":"2022-07-19T10:39:18.877986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'images.zip')","metadata":{"execution":{"iopub.status.busy":"2022-07-19T10:43:06.018612Z","iopub.execute_input":"2022-07-19T10:43:06.019421Z","iopub.status.idle":"2022-07-19T10:43:06.028647Z","shell.execute_reply.started":"2022-07-19T10:43:06.019353Z","shell.execute_reply":"2022-07-19T10:43:06.027807Z"},"trusted":true},"execution_count":null,"outputs":[]}]}