{"cells":[{"metadata":{"trusted":true,"_uuid":"f5c4cb64fd9a397ae679599a68cf72c929d06d39"},"cell_type":"code","source":"import requests\nimport pandas as pd\nfrom bs4 import BeautifulSoup\nimport numpy as np\nimport re\nimport os\nfrom PIL import Image\nimport requests\nfrom io import BytesIO\nfrom joblib import Parallel, delayed\n\nfrom glob import glob\nfrom tqdm import tqdm_notebook\n\nfrom itertools import chain\nfrom collections import Counter\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#newpath = 'external_data' \n#if not os.path.exists(newpath):\n#    os.makedirs(newpath)\n\n\ndef get_html(url):\n    response = requests.get(url)\n    return response.text\n#from https://www.proteinatlas.org/download/subcellular_location.tsv.zip\nsubcellular_location = pd.read_csv('../input/external-data-for-protein-atlas/subcellular_location.tsv', sep=\"\\t\",index_col = None)\n\n\n#get urls\nurls = []\nfor name in subcellular_location[['Gene', 'Gene name']].values:  \n    name = '-'.join(name)\n    url = ('https://www.proteinatlas.org/'+name+'/antibody#ICC')\n    urls.append(url)\n    \ndef load_img(url):\n    html = get_html(url)\n    soup = BeautifulSoup(html, 'lxml')\n\n    links = []\n    for a in soup.findAll('a',{'class':'colorbox'},href=True):\n        if '_selected' in  a['href']:\n            links.append(''.join(('https://www.proteinatlas.org'+a['href']).split('_medium')))\n    \n    i = 0\n    for link in set(links):\n        try:\n            name = url.split('/')[-2]        \n            response = requests.get(link)\n            img = Image.open(BytesIO(response.content))\n            if np.array(img)[:,:,0].mean()<70:\n                img.save('external_data/'+name+'_'+str(i)+'.png')\n                i+=1\n                #break\n        except:\n            pass\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d10768ce112009d2606ce71c0df88caf3ee6c89"},"cell_type":"code","source":"subcellular_location.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea6c9aab5bc390982c05da34cf1f3f42adf57194"},"cell_type":"code","source":"#pages with images\nurls[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35c6d406772d3e7b0e3b55bba98272f61eb49d42"},"cell_type":"code","source":"#load \n\n#num_cores = 12\n#Parallel(n_jobs=num_cores, prefer=\"threads\")(delayed(load_img)(i) for i in urls)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36eb77809052c0a912c8be63404e3e28cf2a2f68"},"cell_type":"code","source":"label_names = {\n    0:  \"Nucleoplasm\", \n    1:  \"Nuclear membrane\",   \n    2:  \"Nucleoli\",   \n    3:  \"Nucleoli fibrillar center\" ,  \n    4:  \"Nuclear speckles\",\n    5:  \"Nuclear bodies\",\n    6:  \"Endoplasmic reticulum\",   \n    7:  \"Golgi apparatus\",\n    8:  \"Peroxisomes\",\n    9:  \"Endosomes\",\n    10:  \"Lysosomes\",\n    11:  \"Intermediate filaments\",   \n    12:  \"Actin filaments\",\n    13:  \"Focal adhesion sites\",   \n    14:  \"Microtubules\",\n    15:  \"Microtubule ends\",   \n    16:  \"Cytokinetic bridge\",   \n    17:  \"Mitotic spindle\",\n    18:  \"Microtubule organizing center\",  \n    19:  \"Centrosome\",\n    20:  \"Lipid droplets\",   \n    21:  \"Plasma membrane\",   \n    22:  \"Cell junctions\", \n    23:  \"Mitochondria\",\n    24:  \"Aggresome\",\n    25:  \"Cytosol\",\n    26:  \"Cytoplasmic bodies\",   \n    27:  \"Rods & rings\"\n  }\n\nall_label_names = {\n    0:  \"Nucleoplasm\", \n    1:  \"Nuclear membrane\",   \n    2:  \"Nucleoli\",   \n    3:  \"Nucleoli fibrillar center\" ,  \n    4:  \"Nuclear speckles\",\n    5:  \"Nuclear bodies\",\n    6:  \"Endoplasmic reticulum\",   \n    7:  \"Golgi apparatus\",\n    8:  \"Peroxisomes\",\n    9:  \"Endosomes\",\n    10:  \"Lysosomes\",\n    11:  \"Intermediate filaments\",   \n    12:  \"Actin filaments\",\n    13:  \"Focal adhesion sites\",   \n    14:  \"Microtubules\",\n    15:  \"Microtubule ends\",   \n    16:  \"Cytokinetic bridge\",   \n    17:  \"Mitotic spindle\",\n    18:  \"Microtubule organizing center\",  \n    19:  \"Centrosome\",\n    20:  \"Lipid droplets\",   \n    21:  \"Plasma membrane\",   \n    22:  \"Cell junctions\", \n    23:  \"Mitochondria\",\n    24:  \"Aggresome\",\n    25:  \"Cytosol\",\n    26:  \"Cytoplasmic bodies\",   \n    27:  \"Rods & rings\",\n    # new classes          \n    28: \"Vesicles\",\n    29: \"Nucleus\",\n    30: \"Midbody\",\n    31: \"Cell Junctions\",\n    32: \"Midbody ring\",\n    33: \"Cleavage furrow\"\n         }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"571222d59e4afd80917220f39b0bd837630e63c0"},"cell_type":"code","source":"all_names = []\nfor j in tqdm_notebook(range(len(subcellular_location))):\n    names = np.array(subcellular_location[['Enhanced','Supported','Approved','Uncertain']].values[j])\n    names = [name for name in names if str(name) != 'nan']\n    split_names = []\n    for i in range(len(names)):\n        split_names = split_names +(names[i].split(';'))\n\n    all_names.append(split_names)\n    \nsubcellular_location['names'] = all_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb52c9de01181818a0fb8b7fafeacc197aa8875b"},"cell_type":"code","source":"from PIL import Image\n\nimg = Image.open(glob('../input/external-data-for-protein-atlas/external_data/*')[0])\nprint(img.size)\nimg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b6b2fb6c4884fb32ae7cfb7034f2bb0273a00c5"},"cell_type":"code","source":"#only old names\ndata = []\nfor i in tqdm_notebook(range(len(subcellular_location))):\n    im_name  = subcellular_location['Gene'].values[i]+'-'+subcellular_location['Gene name'].values[i]\n    for im in glob('../input/external-data-for-protein-atlas/external_data/'+im_name+'*'):\n        labels = []\n        for name in subcellular_location['names'].values[i]:\n            try:\n                if name == 'Rods & Rings': name = \"Rods & rings\"\n                labels.append(list(label_names.values()).index(name))          \n            except:\n                pass\n        if len(labels)>0:\n            data.append([im.split('/')[-1].split('.png')[0], subcellular_location['names'].values[i], labels])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e9cc57279b86e17ebc47f03a9e57c781cb3ec0f"},"cell_type":"code","source":"df = pd.DataFrame(data, columns = ['id', 'names', 'labels'])\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c93abcef6a0b0c765f362a3f161f6323712ce273"},"cell_type":"code","source":"len(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"787dfb4e414828ed73a593dc8b88370b4590e36b"},"cell_type":"code","source":"count_labels = Counter(list(chain.from_iterable(df['labels'].values)))\nplt.figure(figsize = (16,10))\nplt.bar(list(label_names), [count_labels[k] for k in list(label_names)],)\nplt.xticks(list(label_names),list(label_names.values()), rotation=90, size = 15)\n\nfor i in count_labels:\n    plt.text(i-0.4,count_labels[i], count_labels[i], size =12)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e65d4b42e0e0960eeba2210145737253ec338e2"},"cell_type":"code","source":"# download\n# https://www.kaggle.com/artemtprv/external-data-for-protein-atlas","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}