{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-06T13:36:51.045536Z","iopub.execute_input":"2022-12-06T13:36:51.045826Z","iopub.status.idle":"2022-12-06T13:36:51.079048Z","shell.execute_reply.started":"2022-12-06T13:36:51.045763Z","shell.execute_reply":"2022-12-06T13:36:51.078142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Protein-protein interactions graph neighbors","metadata":{}},{"cell_type":"code","source":"fn = '/kaggle/input/ppisim100neighbors/BIOGRID-ALL-4.3.195.tab3.txt'\ndf_ppi = pd.read_csv(fn, sep ='\\t')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:36:51.082247Z","iopub.execute_input":"2022-12-06T13:36:51.082529Z","iopub.status.idle":"2022-12-06T13:37:14.718584Z","shell.execute_reply.started":"2022-12-06T13:36:51.082503Z","shell.execute_reply":"2022-12-06T13:37:14.717569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi['Entrez Gene Interactor A'].replace('-', np.nan, inplace=True)\n\ndf_ppi['Entrez Gene Interactor A'].replace('nan', np.nan, inplace=True)\n\ndf_ppi = df_ppi[df_ppi['Entrez Gene Interactor A'].notna()]\n\ndf_ppi = df_ppi[df_ppi['Entrez Gene Interactor A'].notnull()]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:14.720437Z","iopub.execute_input":"2022-12-06T13:37:14.720905Z","iopub.status.idle":"2022-12-06T13:37:17.256825Z","shell.execute_reply.started":"2022-12-06T13:37:14.720867Z","shell.execute_reply":"2022-12-06T13:37:17.255836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi['Entrez Gene Interactor B'].replace('-', np.nan, inplace=True)\n\ndf_ppi['Entrez Gene Interactor B'].replace('nan', np.nan, inplace=True)\n\ndf_ppi = df_ppi[df_ppi['Entrez Gene Interactor B'].notna()]\n\ndf_ppi = df_ppi[df_ppi['Entrez Gene Interactor B'].notnull()]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:17.258389Z","iopub.execute_input":"2022-12-06T13:37:17.258763Z","iopub.status.idle":"2022-12-06T13:37:19.762601Z","shell.execute_reply.started":"2022-12-06T13:37:17.258724Z","shell.execute_reply":"2022-12-06T13:37:19.761596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_info = pd.read_csv('/kaggle/input/ppisim100neighbors/genes_information_mmscel_challenge_multiome_subtask_tf_info.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:19.765419Z","iopub.execute_input":"2022-12-06T13:37:19.765834Z","iopub.status.idle":"2022-12-06T13:37:20.070795Z","shell.execute_reply.started":"2022-12-06T13:37:19.765795Z","shell.execute_reply":"2022-12-06T13:37:20.069814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_info[\"entrezgene\"].replace('', np.nan, inplace=True)\n\ndf_info[\"entrezgene\"].replace('nan', np.nan, inplace=True)\n\ndf_info = df_info[df_info[\"entrezgene\"].notna()]\n\ndf_info = df_info[df_info[\"entrezgene\"].notnull()]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:20.072294Z","iopub.execute_input":"2022-12-06T13:37:20.072696Z","iopub.status.idle":"2022-12-06T13:37:20.095692Z","shell.execute_reply.started":"2022-12-06T13:37:20.072651Z","shell.execute_reply":"2022-12-06T13:37:20.094765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_info['entrezgene'] = df_info['entrezgene'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:20.097909Z","iopub.execute_input":"2022-12-06T13:37:20.098655Z","iopub.status.idle":"2022-12-06T13:37:20.105117Z","shell.execute_reply.started":"2022-12-06T13:37:20.098617Z","shell.execute_reply":"2022-12-06T13:37:20.104006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import networkx as nx","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:20.106922Z","iopub.execute_input":"2022-12-06T13:37:20.107558Z","iopub.status.idle":"2022-12-06T13:37:20.321811Z","shell.execute_reply.started":"2022-12-06T13:37:20.107517Z","shell.execute_reply":"2022-12-06T13:37:20.320853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"M = nx.from_pandas_edgelist(df_ppi, source='Entrez Gene Interactor A', target='Entrez Gene Interactor B', create_using=nx.MultiGraph())","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:20.323515Z","iopub.execute_input":"2022-12-06T13:37:20.323900Z","iopub.status.idle":"2022-12-06T13:37:34.624252Z","shell.execute_reply.started":"2022-12-06T13:37:20.323864Z","shell.execute_reply":"2022-12-06T13:37:34.623268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(nx.info(M))","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:34.625608Z","iopub.execute_input":"2022-12-06T13:37:34.626648Z","iopub.status.idle":"2022-12-06T13:37:36.096762Z","shell.execute_reply.started":"2022-12-06T13:37:34.626605Z","shell.execute_reply":"2022-12-06T13:37:36.095661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"G = nx.Graph()\nfor u,v,data in M.edges(data=True):\n    w = data['weight'] if 'weight' in data else 1.0\n    if G.has_edge(u,v):\n        G[u][v]['weight'] += w\n    else:\n        G.add_edge(u, v, weight=w)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:36.101177Z","iopub.execute_input":"2022-12-06T13:37:36.101475Z","iopub.status.idle":"2022-12-06T13:37:44.187446Z","shell.execute_reply.started":"2022-12-06T13:37:36.101441Z","shell.execute_reply":"2022-12-06T13:37:44.186411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(nx.info(G))","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:44.188897Z","iopub.execute_input":"2022-12-06T13:37:44.189387Z","iopub.status.idle":"2022-12-06T13:37:44.371853Z","shell.execute_reply.started":"2022-12-06T13:37:44.189347Z","shell.execute_reply":"2022-12-06T13:37:44.370166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_words = pd.read_excel('/kaggle/input/ppisim100neighbors/TotalSeq_B_Universal_Cocktail_v1_140_Antibodies_399904_Barcodes.xlsx')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:44.373444Z","iopub.execute_input":"2022-12-06T13:37:44.373848Z","iopub.status.idle":"2022-12-06T13:37:44.621919Z","shell.execute_reply.started":"2022-12-06T13:37:44.373809Z","shell.execute_reply":"2022-12-06T13:37:44.621038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_words[\"Ensemble ID\"].replace('', np.nan, inplace=True)\n\ndf_words[\"Ensemble ID\"].replace('nan', np.nan, inplace=True)\n\ndf_words = df_words[df_words[\"Ensemble ID\"].notna()]\n\ndf_words = df_words[df_words[\"Ensemble ID\"].notnull()]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:44.623352Z","iopub.execute_input":"2022-12-06T13:37:44.623685Z","iopub.status.idle":"2022-12-06T13:37:44.632050Z","shell.execute_reply.started":"2022-12-06T13:37:44.623648Z","shell.execute_reply":"2022-12-06T13:37:44.631025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_words_list = list(set(df_words['Ensemble ID']))","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:44.633571Z","iopub.execute_input":"2022-12-06T13:37:44.634159Z","iopub.status.idle":"2022-12-06T13:37:44.641878Z","shell.execute_reply.started":"2022-12-06T13:37:44.634123Z","shell.execute_reply":"2022-12-06T13:37:44.640870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install mygene","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:44.643326Z","iopub.execute_input":"2022-12-06T13:37:44.643868Z","iopub.status.idle":"2022-12-06T13:37:56.114350Z","shell.execute_reply.started":"2022-12-06T13:37:44.643832Z","shell.execute_reply":"2022-12-06T13:37:56.112990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mygene","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:56.116222Z","iopub.execute_input":"2022-12-06T13:37:56.117429Z","iopub.status.idle":"2022-12-06T13:37:56.132505Z","shell.execute_reply.started":"2022-12-06T13:37:56.117387Z","shell.execute_reply":"2022-12-06T13:37:56.131422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mg = mygene.MyGeneInfo()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:56.134008Z","iopub.execute_input":"2022-12-06T13:37:56.134386Z","iopub.status.idle":"2022-12-06T13:37:56.141473Z","shell.execute_reply.started":"2022-12-06T13:37:56.134350Z","shell.execute_reply":"2022-12-06T13:37:56.138129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_human = mg.querymany(df_words['Ensemble ID'], scopes='ensembl.gene', fields='entrezgene', species='human')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:56.144739Z","iopub.execute_input":"2022-12-06T13:37:56.145011Z","iopub.status.idle":"2022-12-06T13:37:57.735552Z","shell.execute_reply.started":"2022-12-06T13:37:56.144987Z","shell.execute_reply":"2022-12-06T13:37:57.734528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd_list = [d['entrezgene'] for d in out_human if 'entrezgene' in d]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:57.737196Z","iopub.execute_input":"2022-12-06T13:37:57.737574Z","iopub.status.idle":"2022-12-06T13:37:57.743027Z","shell.execute_reply.started":"2022-12-06T13:37:57.737537Z","shell.execute_reply":"2022-12-06T13:37:57.741901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:57.744990Z","iopub.execute_input":"2022-12-06T13:37:57.745449Z","iopub.status.idle":"2022-12-06T13:37:57.753838Z","shell.execute_reply.started":"2022-12-06T13:37:57.745406Z","shell.execute_reply":"2022-12-06T13:37:57.752794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neigbors_list = []\n\nfor word in tqdm(cd_list):\n    try:\n        neigbors = nx.single_source_shortest_path_length(G, word)\n        neigbors_100 = [n for n in neigbors][:100]\n        neigbors_list.append([word, neigbors_100])\n    except:\n        continue","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:37:57.755397Z","iopub.execute_input":"2022-12-06T13:37:57.755792Z","iopub.status.idle":"2022-12-06T13:38:39.272361Z","shell.execute_reply.started":"2022-12-06T13:37:57.755758Z","shell.execute_reply":"2022-12-06T13:38:39.271238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_neighbors = pd.DataFrame(neigbors_list, columns = ['ppi_cd', 'ppi_neighbors_100'])","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:39.273729Z","iopub.execute_input":"2022-12-06T13:38:39.274548Z","iopub.status.idle":"2022-12-06T13:38:39.281782Z","shell.execute_reply.started":"2022-12-06T13:38:39.274509Z","shell.execute_reply":"2022-12-06T13:38:39.280572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_gb_sep = pd.DataFrame(df_ppi_neighbors['ppi_neighbors_100'].values.tolist()).add_prefix('ppi_neighbor_')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:39.283327Z","iopub.execute_input":"2022-12-06T13:38:39.284982Z","iopub.status.idle":"2022-12-06T13:38:39.296595Z","shell.execute_reply.started":"2022-12-06T13:38:39.284955Z","shell.execute_reply":"2022-12-06T13:38:39.295746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_gb_sep['ppi_cd'] = df_gb_sep['ppi_neighbor_0']\n\ncols = df_gb_sep.columns.tolist()\n\ncols = cols[-1:] + cols[:-1]\n\ndf_gb_sep = df_gb_sep[cols]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:39.298050Z","iopub.execute_input":"2022-12-06T13:38:39.298608Z","iopub.status.idle":"2022-12-06T13:38:39.311250Z","shell.execute_reply.started":"2022-12-06T13:38:39.298572Z","shell.execute_reply":"2022-12-06T13:38:39.310362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_gb_sep = df_gb_sep.astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:39.313094Z","iopub.execute_input":"2022-12-06T13:38:39.313852Z","iopub.status.idle":"2022-12-06T13:38:39.321354Z","shell.execute_reply.started":"2022-12-06T13:38:39.313775Z","shell.execute_reply":"2022-12-06T13:38:39.320483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_names_list = [df_gb_sep[x].values.tolist() for x in df_gb_sep.columns]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:39.323041Z","iopub.execute_input":"2022-12-06T13:38:39.323491Z","iopub.status.idle":"2022-12-06T13:38:39.334740Z","shell.execute_reply.started":"2022-12-06T13:38:39.323406Z","shell.execute_reply":"2022-12-06T13:38:39.333753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_names_flat_list = set([item for sublist in all_names_list for item in sublist])","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:39.337929Z","iopub.execute_input":"2022-12-06T13:38:39.338818Z","iopub.status.idle":"2022-12-06T13:38:39.344868Z","shell.execute_reply.started":"2022-12-06T13:38:39.338779Z","shell.execute_reply":"2022-12-06T13:38:39.344011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_names_out = mg.querymany(all_names_flat_list, scopes='entrezgene', fields='symbol', species='human')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:39.355824Z","iopub.execute_input":"2022-12-06T13:38:39.356077Z","iopub.status.idle":"2022-12-06T13:38:47.843761Z","shell.execute_reply.started":"2022-12-06T13:38:39.356053Z","shell.execute_reply":"2022-12-06T13:38:47.842177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all_names_out = pd.DataFrame(all_names_out, columns=['query', '_id', '_score', 'symbol'])","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:47.845279Z","iopub.execute_input":"2022-12-06T13:38:47.845644Z","iopub.status.idle":"2022-12-06T13:38:47.853250Z","shell.execute_reply.started":"2022-12-06T13:38:47.845606Z","shell.execute_reply":"2022-12-06T13:38:47.851977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all_names_out['query'] = df_all_names_out['query'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:47.854741Z","iopub.execute_input":"2022-12-06T13:38:47.855341Z","iopub.status.idle":"2022-12-06T13:38:47.864959Z","shell.execute_reply.started":"2022-12-06T13:38:47.855305Z","shell.execute_reply":"2022-12-06T13:38:47.863962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = list(df_gb_sep.columns)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:47.867069Z","iopub.execute_input":"2022-12-06T13:38:47.867670Z","iopub.status.idle":"2022-12-06T13:38:47.874565Z","shell.execute_reply.started":"2022-12-06T13:38:47.867636Z","shell.execute_reply":"2022-12-06T13:38:47.873603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_lookup = dict(zip(df_all_names_out['query'], df_all_names_out['symbol']))","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:47.876910Z","iopub.execute_input":"2022-12-06T13:38:47.877214Z","iopub.status.idle":"2022-12-06T13:38:47.889091Z","shell.execute_reply.started":"2022-12-06T13:38:47.877180Z","shell.execute_reply":"2022-12-06T13:38:47.887925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in df_gb_sep.columns:\n    try:        \n        df_gb_sep[column] = [dict_lookup[item] for item in list(df_gb_sep[column])]\n    except:\n        continue","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:47.890471Z","iopub.execute_input":"2022-12-06T13:38:47.891366Z","iopub.status.idle":"2022-12-06T13:38:47.930117Z","shell.execute_reply.started":"2022-12-06T13:38:47.891330Z","shell.execute_reply":"2022-12-06T13:38:47.929209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_gb_sep.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:47.931296Z","iopub.execute_input":"2022-12-06T13:38:47.932304Z","iopub.status.idle":"2022-12-06T13:38:47.942177Z","shell.execute_reply.started":"2022-12-06T13:38:47.932268Z","shell.execute_reply":"2022-12-06T13:38:47.940260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Similarity 100 neighbors","metadata":{}},{"cell_type":"code","source":"df_info['ensembl_id'] = df_info['ensembl_id'].astype(str)\ndf_info['summary'] = df_info['summary'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:47.943865Z","iopub.execute_input":"2022-12-06T13:38:47.944456Z","iopub.status.idle":"2022-12-06T13:38:47.959522Z","shell.execute_reply.started":"2022-12-06T13:38:47.944422Z","shell.execute_reply":"2022-12-06T13:38:47.958591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corpus = list(set(df_info['summary']))","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:47.960820Z","iopub.execute_input":"2022-12-06T13:38:47.961444Z","iopub.status.idle":"2022-12-06T13:38:47.972949Z","shell.execute_reply.started":"2022-12-06T13:38:47.961410Z","shell.execute_reply":"2022-12-06T13:38:47.971817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install -U sentence-transformers\n!pip install faiss-gpu","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:38:47.974695Z","iopub.execute_input":"2022-12-06T13:38:47.975059Z","iopub.status.idle":"2022-12-06T13:39:15.275677Z","shell.execute_reply.started":"2022-12-06T13:38:47.975018Z","shell.execute_reply":"2022-12-06T13:39:15.274329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer, util\nimport faiss","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:39:15.277715Z","iopub.execute_input":"2022-12-06T13:39:15.278128Z","iopub.status.idle":"2022-12-06T13:39:17.960250Z","shell.execute_reply.started":"2022-12-06T13:39:15.278073Z","shell.execute_reply":"2022-12-06T13:39:17.959298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SentenceTransformer('all-distilroberta-v1')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:39:17.961782Z","iopub.execute_input":"2022-12-06T13:39:17.962813Z","iopub.status.idle":"2022-12-06T13:39:42.029222Z","shell.execute_reply.started":"2022-12-06T13:39:17.962774Z","shell.execute_reply":"2022-12-06T13:39:42.028054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embeddings = model.encode(corpus, show_progress_bar=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:39:42.031371Z","iopub.execute_input":"2022-12-06T13:39:42.031747Z","iopub.status.idle":"2022-12-06T13:40:13.718461Z","shell.execute_reply.started":"2022-12-06T13:39:42.031709Z","shell.execute_reply":"2022-12-06T13:40:13.717437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### \"converting embeddings to dataframe and save as .csv\" steps","metadata":{}},{"cell_type":"code","source":"df_emb = pd.DataFrame({'summary': corpus, 'embedding': None})\ndf_emb['embedding'] = embeddings.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:13.720197Z","iopub.execute_input":"2022-12-06T13:40:13.720630Z","iopub.status.idle":"2022-12-06T13:40:14.080041Z","shell.execute_reply.started":"2022-12-06T13:40:13.720585Z","shell.execute_reply":"2022-12-06T13:40:14.078950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_columns = pd.DataFrame(df_emb['embedding'].values.tolist())","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:14.081735Z","iopub.execute_input":"2022-12-06T13:40:14.082334Z","iopub.status.idle":"2022-12-06T13:40:15.768426Z","shell.execute_reply.started":"2022-12-06T13:40:14.082294Z","shell.execute_reply":"2022-12-06T13:40:15.767421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new = pd.concat([df_emb, df_emb_columns], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:15.770035Z","iopub.execute_input":"2022-12-06T13:40:15.770655Z","iopub.status.idle":"2022-12-06T13:40:15.801515Z","shell.execute_reply.started":"2022-12-06T13:40:15.770610Z","shell.execute_reply":"2022-12-06T13:40:15.800628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_list = list(df_emb_new['summary'])","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:15.802862Z","iopub.execute_input":"2022-12-06T13:40:15.803315Z","iopub.status.idle":"2022-12-06T13:40:15.811221Z","shell.execute_reply.started":"2022-12-06T13:40:15.803275Z","shell.execute_reply":"2022-12-06T13:40:15.810168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_lookup = dict(zip(df_info['summary'], df_info['ensembl_id']))","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:15.812739Z","iopub.execute_input":"2022-12-06T13:40:15.813190Z","iopub.status.idle":"2022-12-06T13:40:15.828412Z","shell.execute_reply.started":"2022-12-06T13:40:15.813154Z","shell.execute_reply":"2022-12-06T13:40:15.827477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new['ensembl_id'] = [dict_lookup[item] for item in key_list]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:15.829737Z","iopub.execute_input":"2022-12-06T13:40:15.830175Z","iopub.status.idle":"2022-12-06T13:40:15.845674Z","shell.execute_reply.started":"2022-12-06T13:40:15.830138Z","shell.execute_reply":"2022-12-06T13:40:15.844687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new = df_emb_new.drop(columns=['summary', 'embedding'])","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:15.846804Z","iopub.execute_input":"2022-12-06T13:40:15.847181Z","iopub.status.idle":"2022-12-06T13:40:15.879824Z","shell.execute_reply.started":"2022-12-06T13:40:15.847142Z","shell.execute_reply":"2022-12-06T13:40:15.878882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = df_emb_new.columns.tolist()\n\ncols = cols[-1:] + cols[:-1]\n\ndf_emb_new = df_emb_new[cols]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:15.881268Z","iopub.execute_input":"2022-12-06T13:40:15.881638Z","iopub.status.idle":"2022-12-06T13:40:15.911239Z","shell.execute_reply.started":"2022-12-06T13:40:15.881598Z","shell.execute_reply":"2022-12-06T13:40:15.910231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:15.912868Z","iopub.execute_input":"2022-12-06T13:40:15.913284Z","iopub.status.idle":"2022-12-06T13:40:15.942680Z","shell.execute_reply.started":"2022-12-06T13:40:15.913244Z","shell.execute_reply":"2022-12-06T13:40:15.941803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:15.944375Z","iopub.execute_input":"2022-12-06T13:40:15.944907Z","iopub.status.idle":"2022-12-06T13:40:15.951236Z","shell.execute_reply.started":"2022-12-06T13:40:15.944868Z","shell.execute_reply":"2022-12-06T13:40:15.950146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_emb_new.to_csv('./sentence_transformers_embeddings.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:15.952727Z","iopub.execute_input":"2022-12-06T13:40:15.953386Z","iopub.status.idle":"2022-12-06T13:40:25.706239Z","shell.execute_reply.started":"2022-12-06T13:40:15.953350Z","shell.execute_reply":"2022-12-06T13:40:25.705263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### create a faiss-index (flat)","metadata":{}},{"cell_type":"code","source":"faiss.normalize_L2(embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:25.707909Z","iopub.execute_input":"2022-12-06T13:40:25.708414Z","iopub.status.idle":"2022-12-06T13:40:25.725869Z","shell.execute_reply.started":"2022-12-06T13:40:25.708374Z","shell.execute_reply":"2022-12-06T13:40:25.725016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = embeddings.shape[1]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:25.727316Z","iopub.execute_input":"2022-12-06T13:40:25.728340Z","iopub.status.idle":"2022-12-06T13:40:25.733309Z","shell.execute_reply.started":"2022-12-06T13:40:25.728304Z","shell.execute_reply":"2022-12-06T13:40:25.732059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = faiss.IndexFlatL2(d)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:25.735090Z","iopub.execute_input":"2022-12-06T13:40:25.735480Z","iopub.status.idle":"2022-12-06T13:40:25.743687Z","shell.execute_reply.started":"2022-12-06T13:40:25.735421Z","shell.execute_reply":"2022-12-06T13:40:25.742814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index.add(embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:25.745407Z","iopub.execute_input":"2022-12-06T13:40:25.745820Z","iopub.status.idle":"2022-12-06T13:40:25.779892Z","shell.execute_reply.started":"2022-12-06T13:40:25.745786Z","shell.execute_reply":"2022-12-06T13:40:25.779069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def search(query):\n    query_vector = model.encode([query])\n    faiss.normalize_L2(query_vector)\n    k = 100\n    similarities, similarities_ids = index.search(query_vector.astype(np.float32), k)\n    similarities = np.around(np.clip(similarities, 0, 1), decimals=4)\n    highest_val_idx = np.argmax([sum(sim_vec) for sim_vec in similarities])\n    l1 = [corpus[_id] for _id in similarities_ids.tolist()[0]]\n    l2 = similarities[highest_val_idx].tolist()\n    return list(zip(l1,l2))","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:25.782134Z","iopub.execute_input":"2022-12-06T13:40:25.782724Z","iopub.status.idle":"2022-12-06T13:40:25.789454Z","shell.execute_reply.started":"2022-12-06T13:40:25.782689Z","shell.execute_reply":"2022-12-06T13:40:25.788570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n\nedgelist = []\nfor word in tqdm(corpus):\n    edges = []\n    for s, w in search(word):\n        edges.extend((word, s, w))\n    edgelist.append(edges)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:40:25.791127Z","iopub.execute_input":"2022-12-06T13:40:25.791538Z","iopub.status.idle":"2022-12-06T13:49:35.483937Z","shell.execute_reply.started":"2022-12-06T13:40:25.791499Z","shell.execute_reply":"2022-12-06T13:49:35.483068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flat_edgelist = [item for sublist in edgelist for item in sublist]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:35.485280Z","iopub.execute_input":"2022-12-06T13:49:35.485742Z","iopub.status.idle":"2022-12-06T13:49:35.638499Z","shell.execute_reply.started":"2022-12-06T13:49:35.485704Z","shell.execute_reply":"2022-12-06T13:49:35.637460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"edgelist_3 = [list(flat_edgelist[i:i+3]) for i in range(0, len(flat_edgelist), 3)]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:35.639995Z","iopub.execute_input":"2022-12-06T13:49:35.640371Z","iopub.status.idle":"2022-12-06T13:49:38.681735Z","shell.execute_reply.started":"2022-12-06T13:49:35.640333Z","shell.execute_reply":"2022-12-06T13:49:38.680738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors = pd.DataFrame(edgelist_3, columns = ['Id1', 'Id2', 'weight'])","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:38.683093Z","iopub.execute_input":"2022-12-06T13:49:38.683493Z","iopub.status.idle":"2022-12-06T13:49:38.972948Z","shell.execute_reply.started":"2022-12-06T13:49:38.683446Z","shell.execute_reply":"2022-12-06T13:49:38.971966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_list = list(df_neighbors['Id1'])\nkey_list_2 = list(df_neighbors['Id2'])","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:38.974472Z","iopub.execute_input":"2022-12-06T13:49:38.974852Z","iopub.status.idle":"2022-12-06T13:49:39.141501Z","shell.execute_reply.started":"2022-12-06T13:49:38.974815Z","shell.execute_reply":"2022-12-06T13:49:39.140479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_lookup = dict(zip(df_info['summary'], df_info['ensembl_id']))","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:39.143028Z","iopub.execute_input":"2022-12-06T13:49:39.143686Z","iopub.status.idle":"2022-12-06T13:49:39.156095Z","shell.execute_reply.started":"2022-12-06T13:49:39.143649Z","shell.execute_reply":"2022-12-06T13:49:39.155183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors['Id1'] = [dict_lookup[item] for item in key_list]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:39.157378Z","iopub.execute_input":"2022-12-06T13:49:39.158062Z","iopub.status.idle":"2022-12-06T13:49:39.280320Z","shell.execute_reply.started":"2022-12-06T13:49:39.158019Z","shell.execute_reply":"2022-12-06T13:49:39.279323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors['Id2'] = [dict_lookup[item] for item in key_list_2]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:39.281576Z","iopub.execute_input":"2022-12-06T13:49:39.281921Z","iopub.status.idle":"2022-12-06T13:49:39.452041Z","shell.execute_reply.started":"2022-12-06T13:49:39.281885Z","shell.execute_reply":"2022-12-06T13:49:39.451084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group_df_neighbors = df_neighbors.groupby('Id1')\n\nneighbors_list = group_df_neighbors.apply(lambda x: x['Id2'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:39.453378Z","iopub.execute_input":"2022-12-06T13:49:39.454245Z","iopub.status.idle":"2022-12-06T13:49:40.565580Z","shell.execute_reply.started":"2022-12-06T13:49:39.454205Z","shell.execute_reply":"2022-12-06T13:49:40.564445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors_list = pd.DataFrame(neighbors_list, columns = ['sim_neighbors']).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.567282Z","iopub.execute_input":"2022-12-06T13:49:40.567677Z","iopub.status.idle":"2022-12-06T13:49:40.574839Z","shell.execute_reply.started":"2022-12-06T13:49:40.567635Z","shell.execute_reply":"2022-12-06T13:49:40.573681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_neighbors_100 = pd.DataFrame(df_neighbors_list['sim_neighbors'].values.tolist()).add_prefix('sim_neighbor_')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.576527Z","iopub.execute_input":"2022-12-06T13:49:40.576895Z","iopub.status.idle":"2022-12-06T13:49:40.688829Z","shell.execute_reply.started":"2022-12-06T13:49:40.576860Z","shell.execute_reply":"2022-12-06T13:49:40.687850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_neighbors_100['sim_cd'] = df_sim_neighbors_100['sim_neighbor_0']","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.690425Z","iopub.execute_input":"2022-12-06T13:49:40.690846Z","iopub.status.idle":"2022-12-06T13:49:40.696483Z","shell.execute_reply.started":"2022-12-06T13:49:40.690805Z","shell.execute_reply":"2022-12-06T13:49:40.695438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = df_sim_neighbors_100.columns.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.698152Z","iopub.execute_input":"2022-12-06T13:49:40.698915Z","iopub.status.idle":"2022-12-06T13:49:40.706746Z","shell.execute_reply.started":"2022-12-06T13:49:40.698873Z","shell.execute_reply":"2022-12-06T13:49:40.705869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = cols[-1:] + cols[:-1]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.708269Z","iopub.execute_input":"2022-12-06T13:49:40.708697Z","iopub.status.idle":"2022-12-06T13:49:40.715912Z","shell.execute_reply.started":"2022-12-06T13:49:40.708659Z","shell.execute_reply":"2022-12-06T13:49:40.715019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_neighbors_100 = df_sim_neighbors_100[cols]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.717490Z","iopub.execute_input":"2022-12-06T13:49:40.717911Z","iopub.status.idle":"2022-12-06T13:49:40.752666Z","shell.execute_reply.started":"2022-12-06T13:49:40.717877Z","shell.execute_reply":"2022-12-06T13:49:40.751667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_neighbors_100 = df_sim_neighbors_100[df_sim_neighbors_100['sim_cd'].isin(df_words_list)]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.754084Z","iopub.execute_input":"2022-12-06T13:49:40.754498Z","iopub.status.idle":"2022-12-06T13:49:40.766092Z","shell.execute_reply.started":"2022-12-06T13:49:40.754461Z","shell.execute_reply":"2022-12-06T13:49:40.764979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_neighbors_100.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.768010Z","iopub.execute_input":"2022-12-06T13:49:40.768445Z","iopub.status.idle":"2022-12-06T13:49:40.795705Z","shell.execute_reply.started":"2022-12-06T13:49:40.768410Z","shell.execute_reply":"2022-12-06T13:49:40.794048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_neighbors_100 = df_sim_neighbors_100.astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.797975Z","iopub.execute_input":"2022-12-06T13:49:40.799070Z","iopub.status.idle":"2022-12-06T13:49:40.804004Z","shell.execute_reply.started":"2022-12-06T13:49:40.798987Z","shell.execute_reply":"2022-12-06T13:49:40.803419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_names_list = [df_sim_neighbors_100[x].values.tolist() for x in df_sim_neighbors_100.columns]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.805302Z","iopub.execute_input":"2022-12-06T13:49:40.805819Z","iopub.status.idle":"2022-12-06T13:49:40.817706Z","shell.execute_reply.started":"2022-12-06T13:49:40.805785Z","shell.execute_reply":"2022-12-06T13:49:40.816777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_names_flat_list = set([item for sublist in all_names_list for item in sublist])","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.819399Z","iopub.execute_input":"2022-12-06T13:49:40.819874Z","iopub.status.idle":"2022-12-06T13:49:40.826981Z","shell.execute_reply.started":"2022-12-06T13:49:40.819799Z","shell.execute_reply":"2022-12-06T13:49:40.826087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_names_out = mg.querymany(all_names_flat_list, scopes='ensembl.gene', fields='symbol', species='human')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:40.828634Z","iopub.execute_input":"2022-12-06T13:49:40.829093Z","iopub.status.idle":"2022-12-06T13:49:47.355795Z","shell.execute_reply.started":"2022-12-06T13:49:40.829056Z","shell.execute_reply":"2022-12-06T13:49:47.354766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all_names_out = pd.DataFrame(all_names_out, columns=['query', '_id', '_score', 'symbol'])","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:47.357142Z","iopub.execute_input":"2022-12-06T13:49:47.358394Z","iopub.status.idle":"2022-12-06T13:49:47.366579Z","shell.execute_reply.started":"2022-12-06T13:49:47.358355Z","shell.execute_reply":"2022-12-06T13:49:47.365408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all_names_out['query'] = df_all_names_out['query'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:47.368342Z","iopub.execute_input":"2022-12-06T13:49:47.368756Z","iopub.status.idle":"2022-12-06T13:49:47.378375Z","shell.execute_reply.started":"2022-12-06T13:49:47.368721Z","shell.execute_reply":"2022-12-06T13:49:47.377400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_lookup = dict(zip(df_all_names_out['query'], df_all_names_out['symbol']))","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:47.379979Z","iopub.execute_input":"2022-12-06T13:49:47.380449Z","iopub.status.idle":"2022-12-06T13:49:47.390299Z","shell.execute_reply.started":"2022-12-06T13:49:47.380413Z","shell.execute_reply":"2022-12-06T13:49:47.389164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in df_sim_neighbors_100.columns:\n    try:        \n        df_sim_neighbors_100[column] = [dict_lookup[item] for item in list(df_sim_neighbors_100[column])]\n    except:\n        continue","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:47.391836Z","iopub.execute_input":"2022-12-06T13:49:47.392883Z","iopub.status.idle":"2022-12-06T13:49:47.423204Z","shell.execute_reply.started":"2022-12-06T13:49:47.392847Z","shell.execute_reply":"2022-12-06T13:49:47.422368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_neighbors_100.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:47.435751Z","iopub.execute_input":"2022-12-06T13:49:47.436640Z","iopub.status.idle":"2022-12-06T13:49:47.460211Z","shell.execute_reply.started":"2022-12-06T13:49:47.436605Z","shell.execute_reply":"2022-12-06T13:49:47.459160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation between PPI and Sim neighbors","metadata":{}},{"cell_type":"code","source":"df_ppi = df_gb_sep","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:47.461858Z","iopub.execute_input":"2022-12-06T13:49:47.462261Z","iopub.status.idle":"2022-12-06T13:49:47.996417Z","shell.execute_reply.started":"2022-12-06T13:49:47.462198Z","shell.execute_reply":"2022-12-06T13:49:47.995292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi.to_csv('./df_ppi_100_neighbors.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:47.998360Z","iopub.execute_input":"2022-12-06T13:49:47.998967Z","iopub.status.idle":"2022-12-06T13:49:48.015118Z","shell.execute_reply.started":"2022-12-06T13:49:47.998921Z","shell.execute_reply":"2022-12-06T13:49:48.014277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.016337Z","iopub.execute_input":"2022-12-06T13:49:48.016761Z","iopub.status.idle":"2022-12-06T13:49:48.046033Z","shell.execute_reply.started":"2022-12-06T13:49:48.016724Z","shell.execute_reply":"2022-12-06T13:49:48.044989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim = df_sim_neighbors_100","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.047520Z","iopub.execute_input":"2022-12-06T13:49:48.048143Z","iopub.status.idle":"2022-12-06T13:49:48.054334Z","shell.execute_reply.started":"2022-12-06T13:49:48.048089Z","shell.execute_reply":"2022-12-06T13:49:48.053319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim.to_csv('./df_sim_100_neighbors.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.055798Z","iopub.execute_input":"2022-12-06T13:49:48.056165Z","iopub.status.idle":"2022-12-06T13:49:48.067432Z","shell.execute_reply.started":"2022-12-06T13:49:48.056123Z","shell.execute_reply":"2022-12-06T13:49:48.066426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlations for Similarity 100 neighbors","metadata":{}},{"cell_type":"code","source":"df_sim['neighbors_list'] = df_sim[df_sim.columns[1:]].apply(\n    lambda x: ','.join(x.dropna().astype(str)),\n    axis=1\n)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.068938Z","iopub.execute_input":"2022-12-06T13:49:48.069334Z","iopub.status.idle":"2022-12-06T13:49:48.101949Z","shell.execute_reply.started":"2022-12-06T13:49:48.069298Z","shell.execute_reply":"2022-12-06T13:49:48.101005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim['neighbors_list'] = df_sim['neighbors_list'].str.split(',')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.103489Z","iopub.execute_input":"2022-12-06T13:49:48.103854Z","iopub.status.idle":"2022-12-06T13:49:48.113238Z","shell.execute_reply.started":"2022-12-06T13:49:48.103818Z","shell.execute_reply":"2022-12-06T13:49:48.112313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_cd36 = df_sim[df_sim['sim_cd'] == 'CD36']","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.114959Z","iopub.execute_input":"2022-12-06T13:49:48.115915Z","iopub.status.idle":"2022-12-06T13:49:48.121938Z","shell.execute_reply.started":"2022-12-06T13:49:48.115881Z","shell.execute_reply":"2022-12-06T13:49:48.121331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sim_cd36_list = df_sim_cd36['neighbors_list'].to_numpy().tolist()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.123033Z","iopub.execute_input":"2022-12-06T13:49:48.124012Z","iopub.status.idle":"2022-12-06T13:49:48.130477Z","shell.execute_reply.started":"2022-12-06T13:49:48.123979Z","shell.execute_reply":"2022-12-06T13:49:48.129669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sim_cd36_flat_edgelist = [item for sublist in sim_cd36_list for item in sublist]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.131567Z","iopub.execute_input":"2022-12-06T13:49:48.133345Z","iopub.status.idle":"2022-12-06T13:49:48.140132Z","shell.execute_reply.started":"2022-12-06T13:49:48.133319Z","shell.execute_reply":"2022-12-06T13:49:48.138899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim['intersection'] = (df_sim['neighbors_list']\n                     .apply(lambda x: list(set(x).intersection(set(sim_cd36_flat_edgelist)))\n                           )\n                  )","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.141392Z","iopub.execute_input":"2022-12-06T13:49:48.142457Z","iopub.status.idle":"2022-12-06T13:49:48.151396Z","shell.execute_reply.started":"2022-12-06T13:49:48.142421Z","shell.execute_reply":"2022-12-06T13:49:48.150413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_other = df_sim[df_sim['sim_cd'] != 'CD36']","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.152642Z","iopub.execute_input":"2022-12-06T13:49:48.153127Z","iopub.status.idle":"2022-12-06T13:49:48.162014Z","shell.execute_reply.started":"2022-12-06T13:49:48.153072Z","shell.execute_reply":"2022-12-06T13:49:48.161148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_explode = df_sim_other.explode('intersection')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.163341Z","iopub.execute_input":"2022-12-06T13:49:48.163882Z","iopub.status.idle":"2022-12-06T13:49:48.181391Z","shell.execute_reply.started":"2022-12-06T13:49:48.163844Z","shell.execute_reply":"2022-12-06T13:49:48.180534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_pivot = df_sim_explode.pivot_table(index=\"sim_cd\", values=\"intersection\", aggfunc='count')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.183987Z","iopub.execute_input":"2022-12-06T13:49:48.184269Z","iopub.status.idle":"2022-12-06T13:49:48.195673Z","shell.execute_reply.started":"2022-12-06T13:49:48.184245Z","shell.execute_reply":"2022-12-06T13:49:48.194624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_pivot['sim_cd'] = df_sim_pivot.index","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.197284Z","iopub.execute_input":"2022-12-06T13:49:48.197702Z","iopub.status.idle":"2022-12-06T13:49:48.203900Z","shell.execute_reply.started":"2022-12-06T13:49:48.197666Z","shell.execute_reply":"2022-12-06T13:49:48.202914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.205252Z","iopub.execute_input":"2022-12-06T13:49:48.205792Z","iopub.status.idle":"2022-12-06T13:49:48.272347Z","shell.execute_reply.started":"2022-12-06T13:49:48.205756Z","shell.execute_reply":"2022-12-06T13:49:48.271488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure(num=None, figsize=(10,20), dpi=80, facecolor='w', edgecolor='r')\nsns.barplot(x=\"intersection\", y = \"sim_cd\" , data = df_sim_pivot)\nplt.xticks(rotation=90)\nplt.title(\"Similarity 100 neighbors CD36 intersection\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:48.273867Z","iopub.execute_input":"2022-12-06T13:49:48.274245Z","iopub.status.idle":"2022-12-06T13:49:49.817589Z","shell.execute_reply.started":"2022-12-06T13:49:48.274210Z","shell.execute_reply":"2022-12-06T13:49:49.816680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlations for PPI 100 neighbors","metadata":{}},{"cell_type":"code","source":"df_ppi['neighbors_list'] = df_ppi[df_ppi.columns[1:]].apply(\n    lambda x: ','.join(x.dropna().astype(str)),\n    axis=1\n)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.818694Z","iopub.execute_input":"2022-12-06T13:49:49.818979Z","iopub.status.idle":"2022-12-06T13:49:49.854265Z","shell.execute_reply.started":"2022-12-06T13:49:49.818952Z","shell.execute_reply":"2022-12-06T13:49:49.853396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi['neighbors_list'] = df_ppi['neighbors_list'].str.split(',')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.856704Z","iopub.execute_input":"2022-12-06T13:49:49.857370Z","iopub.status.idle":"2022-12-06T13:49:49.863392Z","shell.execute_reply.started":"2022-12-06T13:49:49.857333Z","shell.execute_reply":"2022-12-06T13:49:49.862506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_cd36 = df_ppi[df_ppi['ppi_cd'] == 'CD36']","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.864830Z","iopub.execute_input":"2022-12-06T13:49:49.865203Z","iopub.status.idle":"2022-12-06T13:49:49.874072Z","shell.execute_reply.started":"2022-12-06T13:49:49.865168Z","shell.execute_reply":"2022-12-06T13:49:49.872997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ppi_cd36_list = df_ppi_cd36['neighbors_list'].to_numpy().tolist()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.875757Z","iopub.execute_input":"2022-12-06T13:49:49.876121Z","iopub.status.idle":"2022-12-06T13:49:49.884250Z","shell.execute_reply.started":"2022-12-06T13:49:49.876072Z","shell.execute_reply":"2022-12-06T13:49:49.883373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ppi_cd36_flat_edgelist = [item for sublist in ppi_cd36_list for item in sublist]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.885889Z","iopub.execute_input":"2022-12-06T13:49:49.886293Z","iopub.status.idle":"2022-12-06T13:49:49.893264Z","shell.execute_reply.started":"2022-12-06T13:49:49.886260Z","shell.execute_reply":"2022-12-06T13:49:49.892276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi['intersection'] = (df_ppi['neighbors_list']\n                     .apply(lambda x: list(set(x).intersection(set(ppi_cd36_flat_edgelist)))\n                           )\n                  )","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.894838Z","iopub.execute_input":"2022-12-06T13:49:49.895207Z","iopub.status.idle":"2022-12-06T13:49:49.904666Z","shell.execute_reply.started":"2022-12-06T13:49:49.895173Z","shell.execute_reply":"2022-12-06T13:49:49.903636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_other = df_ppi[df_ppi['ppi_cd'] != 'CD36']","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.906242Z","iopub.execute_input":"2022-12-06T13:49:49.906604Z","iopub.status.idle":"2022-12-06T13:49:49.915900Z","shell.execute_reply.started":"2022-12-06T13:49:49.906569Z","shell.execute_reply":"2022-12-06T13:49:49.915012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_explode = df_ppi_other.explode('intersection')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.917648Z","iopub.execute_input":"2022-12-06T13:49:49.918014Z","iopub.status.idle":"2022-12-06T13:49:49.928331Z","shell.execute_reply.started":"2022-12-06T13:49:49.917977Z","shell.execute_reply":"2022-12-06T13:49:49.927436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_pivot = df_ppi_explode.pivot_table(index=\"ppi_cd\", values=\"intersection\", aggfunc='count')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.930027Z","iopub.execute_input":"2022-12-06T13:49:49.930609Z","iopub.status.idle":"2022-12-06T13:49:49.941702Z","shell.execute_reply.started":"2022-12-06T13:49:49.930563Z","shell.execute_reply":"2022-12-06T13:49:49.940765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_pivot['ppi_cd'] = df_ppi_pivot.index","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.943457Z","iopub.execute_input":"2022-12-06T13:49:49.943799Z","iopub.status.idle":"2022-12-06T13:49:49.950790Z","shell.execute_reply.started":"2022-12-06T13:49:49.943766Z","shell.execute_reply":"2022-12-06T13:49:49.949778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure(num=None, figsize=(10,20), dpi=80, facecolor='w', edgecolor='r')\nsns.barplot(x=\"intersection\", y = \"ppi_cd\" , data = df_ppi_pivot)\nplt.xticks(rotation=90)\nplt.title(\"PPI 100 neighbors CD36 intersection\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:49.952029Z","iopub.execute_input":"2022-12-06T13:49:49.952533Z","iopub.status.idle":"2022-12-06T13:49:51.020287Z","shell.execute_reply.started":"2022-12-06T13:49:49.952497Z","shell.execute_reply":"2022-12-06T13:49:51.019174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sim - PPI Intersection bt selected cd","metadata":{}},{"cell_type":"code","source":"sim_ppi_cd36 = list(set(sim_cd36_flat_edgelist).intersection(set(ppi_cd36_flat_edgelist)))","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.022196Z","iopub.execute_input":"2022-12-06T13:49:51.022597Z","iopub.status.idle":"2022-12-06T13:49:51.027923Z","shell.execute_reply.started":"2022-12-06T13:49:51.022560Z","shell.execute_reply":"2022-12-06T13:49:51.026707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(sim_ppi_cd36)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.029682Z","iopub.execute_input":"2022-12-06T13:49:51.030455Z","iopub.status.idle":"2022-12-06T13:49:51.041178Z","shell.execute_reply.started":"2022-12-06T13:49:51.030416Z","shell.execute_reply":"2022-12-06T13:49:51.040160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sim_ppi_cd36","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.042751Z","iopub.execute_input":"2022-12-06T13:49:51.043181Z","iopub.status.idle":"2022-12-06T13:49:51.051281Z","shell.execute_reply.started":"2022-12-06T13:49:51.043146Z","shell.execute_reply":"2022-12-06T13:49:51.050088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sim - PPI statistics","metadata":{}},{"cell_type":"code","source":"df_sim_pivot = df_sim_pivot.rename(columns={'sim_cd': 'cd', 'intersection': 'sim_intersection'})","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.053384Z","iopub.execute_input":"2022-12-06T13:49:51.053647Z","iopub.status.idle":"2022-12-06T13:49:51.059818Z","shell.execute_reply.started":"2022-12-06T13:49:51.053613Z","shell.execute_reply":"2022-12-06T13:49:51.058742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_pivot = df_ppi_pivot.rename(columns={'ppi_cd': 'cd', 'intersection': 'ppi_intersection'})","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.061351Z","iopub.execute_input":"2022-12-06T13:49:51.062399Z","iopub.status.idle":"2022-12-06T13:49:51.069629Z","shell.execute_reply.started":"2022-12-06T13:49:51.062365Z","shell.execute_reply":"2022-12-06T13:49:51.068671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_pivot = df_sim_pivot.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.070920Z","iopub.execute_input":"2022-12-06T13:49:51.071601Z","iopub.status.idle":"2022-12-06T13:49:51.078408Z","shell.execute_reply.started":"2022-12-06T13:49:51.071566Z","shell.execute_reply":"2022-12-06T13:49:51.077423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_pivot = df_ppi_pivot.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.080037Z","iopub.execute_input":"2022-12-06T13:49:51.080450Z","iopub.status.idle":"2022-12-06T13:49:51.087200Z","shell.execute_reply.started":"2022-12-06T13:49:51.080414Z","shell.execute_reply":"2022-12-06T13:49:51.086228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_pivot.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.088821Z","iopub.execute_input":"2022-12-06T13:49:51.089232Z","iopub.status.idle":"2022-12-06T13:49:51.100837Z","shell.execute_reply.started":"2022-12-06T13:49:51.089198Z","shell.execute_reply":"2022-12-06T13:49:51.099659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merge = pd.merge(df_sim_pivot, df_ppi_pivot, left_on='cd', right_on='cd')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.102605Z","iopub.execute_input":"2022-12-06T13:49:51.102942Z","iopub.status.idle":"2022-12-06T13:49:51.113057Z","shell.execute_reply.started":"2022-12-06T13:49:51.102908Z","shell.execute_reply":"2022-12-06T13:49:51.111900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merge.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.114687Z","iopub.execute_input":"2022-12-06T13:49:51.115062Z","iopub.status.idle":"2022-12-06T13:49:51.129373Z","shell.execute_reply.started":"2022-12-06T13:49:51.115022Z","shell.execute_reply":"2022-12-06T13:49:51.128269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_ppi_stat = df_merge[['cd', 'sim_intersection', 'ppi_intersection']]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.130927Z","iopub.execute_input":"2022-12-06T13:49:51.131802Z","iopub.status.idle":"2022-12-06T13:49:51.139513Z","shell.execute_reply.started":"2022-12-06T13:49:51.131775Z","shell.execute_reply":"2022-12-06T13:49:51.138660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_ppi_stat.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.141036Z","iopub.execute_input":"2022-12-06T13:49:51.141564Z","iopub.status.idle":"2022-12-06T13:49:51.154032Z","shell.execute_reply.started":"2022-12-06T13:49:51.141531Z","shell.execute_reply":"2022-12-06T13:49:51.153154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sim_ppi_stat.to_csv('./df_sim_ppi_stat.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:49:51.155211Z","iopub.execute_input":"2022-12-06T13:49:51.155783Z","iopub.status.idle":"2022-12-06T13:49:51.162947Z","shell.execute_reply.started":"2022-12-06T13:49:51.155758Z","shell.execute_reply":"2022-12-06T13:49:51.161214Z"},"trusted":true},"execution_count":null,"outputs":[]}]}