{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# О блокноте","metadata":{}},{"cell_type":"markdown","source":"Мы решили провести разведанализ данных для [этого](https://www.biolegend.com/Files/Images/BioLegend/totalseq/TotalSeq_B_Universal_Cocktail_v1_140_Antibodies_399904_Barcodes.xlsx) набора генов, что может пригодиться для решения задания [этого](https://www.kaggle.com/competitions/open-problems-multimodal) конкурса.","metadata":{}},{"cell_type":"markdown","source":"Метод, который мы используем, заключается в получении эмбеддингов (низкоразмерных вложений) описаний генов и нахождения сходства между предложениями. Для этого мы используем предобученную модель 'all-distilroberta-v1' из арсенала [sentence-transformers](https://www.sbert.net/docs/pretrained_models.html).","metadata":{}},{"cell_type":"markdown","source":"# Предобработка данных","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:24.348792Z","iopub.execute_input":"2022-10-14T15:46:24.349227Z","iopub.status.idle":"2022-10-14T15:46:24.406173Z","shell.execute_reply.started":"2022-10-14T15:46:24.349140Z","shell.execute_reply":"2022-10-14T15:46:24.405026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:24.407951Z","iopub.execute_input":"2022-10-14T15:46:24.408399Z","iopub.status.idle":"2022-10-14T15:46:24.415436Z","shell.execute_reply.started":"2022-10-14T15:46:24.408364Z","shell.execute_reply":"2022-10-14T15:46:24.414517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/genes-information/genes_information_mmscel_challenge_multiome_subtask_tf_info.csv')","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:24.416545Z","iopub.execute_input":"2022-10-14T15:46:24.416804Z","iopub.status.idle":"2022-10-14T15:46:24.771795Z","shell.execute_reply.started":"2022-10-14T15:46:24.416780Z","shell.execute_reply":"2022-10-14T15:46:24.770823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:24.774898Z","iopub.execute_input":"2022-10-14T15:46:24.775537Z","iopub.status.idle":"2022-10-14T15:46:24.784675Z","shell.execute_reply.started":"2022-10-14T15:46:24.775498Z","shell.execute_reply":"2022-10-14T15:46:24.783623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:24.786062Z","iopub.execute_input":"2022-10-14T15:46:24.786648Z","iopub.status.idle":"2022-10-14T15:46:24.822011Z","shell.execute_reply.started":"2022-10-14T15:46:24.786609Z","shell.execute_reply":"2022-10-14T15:46:24.820936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['symbol'] = df['symbol'].astype(str)\ndf['name'] = df['name'].astype(str)\ndf['summary'] = df['summary'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:24.823672Z","iopub.execute_input":"2022-10-14T15:46:24.824023Z","iopub.status.idle":"2022-10-14T15:46:24.841911Z","shell.execute_reply.started":"2022-10-14T15:46:24.823985Z","shell.execute_reply":"2022-10-14T15:46:24.841099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"summary\"].replace('', np.nan, inplace=True)\n\ndf[\"summary\"].replace('nan', np.nan, inplace=True)\n\ndf = df[df[\"summary\"].notna()]\n\ndf = df[df[\"summary\"].notnull()]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:24.845056Z","iopub.execute_input":"2022-10-14T15:46:24.845336Z","iopub.status.idle":"2022-10-14T15:46:24.863415Z","shell.execute_reply.started":"2022-10-14T15:46:24.845312Z","shell.execute_reply":"2022-10-14T15:46:24.862593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:24.864900Z","iopub.execute_input":"2022-10-14T15:46:24.865301Z","iopub.status.idle":"2022-10-14T15:46:24.871980Z","shell.execute_reply.started":"2022-10-14T15:46:24.865267Z","shell.execute_reply":"2022-10-14T15:46:24.870460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"url = 'https://www.biolegend.com/Files/Images/BioLegend/totalseq/TotalSeq_B_Universal_Cocktail_v1_140_Antibodies_399904_Barcodes.xlsx'","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:24.873822Z","iopub.execute_input":"2022-10-14T15:46:24.874223Z","iopub.status.idle":"2022-10-14T15:46:24.880130Z","shell.execute_reply.started":"2022-10-14T15:46:24.874188Z","shell.execute_reply":"2022-10-14T15:46:24.878991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_words = pd.read_excel(url)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:24.885412Z","iopub.execute_input":"2022-10-14T15:46:24.885887Z","iopub.status.idle":"2022-10-14T15:46:26.265649Z","shell.execute_reply.started":"2022-10-14T15:46:24.885854Z","shell.execute_reply":"2022-10-14T15:46:26.264655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_words.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.267170Z","iopub.execute_input":"2022-10-14T15:46:26.267750Z","iopub.status.idle":"2022-10-14T15:46:26.280473Z","shell.execute_reply.started":"2022-10-14T15:46:26.267709Z","shell.execute_reply":"2022-10-14T15:46:26.279566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_words.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.281812Z","iopub.execute_input":"2022-10-14T15:46:26.282945Z","iopub.status.idle":"2022-10-14T15:46:26.292894Z","shell.execute_reply.started":"2022-10-14T15:46:26.282909Z","shell.execute_reply":"2022-10-14T15:46:26.291967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"genes_id_list = list(df_words['Ensemble ID'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.294202Z","iopub.execute_input":"2022-10-14T15:46:26.294457Z","iopub.status.idle":"2022-10-14T15:46:26.303390Z","shell.execute_reply.started":"2022-10-14T15:46:26.294432Z","shell.execute_reply":"2022-10-14T15:46:26.302463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_filter = df[df['ensembl_id'].isin(genes_id_list)]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.304955Z","iopub.execute_input":"2022-10-14T15:46:26.305425Z","iopub.status.idle":"2022-10-14T15:46:26.317309Z","shell.execute_reply.started":"2022-10-14T15:46:26.305391Z","shell.execute_reply":"2022-10-14T15:46:26.316498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_filter = df_filter[['symbol', 'summary']]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.318936Z","iopub.execute_input":"2022-10-14T15:46:26.319411Z","iopub.status.idle":"2022-10-14T15:46:26.328627Z","shell.execute_reply.started":"2022-10-14T15:46:26.319377Z","shell.execute_reply":"2022-10-14T15:46:26.327704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_filter.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.329871Z","iopub.execute_input":"2022-10-14T15:46:26.330738Z","iopub.status.idle":"2022-10-14T15:46:26.340957Z","shell.execute_reply.started":"2022-10-14T15:46:26.330704Z","shell.execute_reply":"2022-10-14T15:46:26.340063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_filter.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.343913Z","iopub.execute_input":"2022-10-14T15:46:26.344211Z","iopub.status.idle":"2022-10-14T15:46:26.354962Z","shell.execute_reply.started":"2022-10-14T15:46:26.344186Z","shell.execute_reply":"2022-10-14T15:46:26.354041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_corpus = list(set(df_filter['summary']))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.356551Z","iopub.execute_input":"2022-10-14T15:46:26.357656Z","iopub.status.idle":"2022-10-14T15:46:26.365352Z","shell.execute_reply.started":"2022-10-14T15:46:26.357622Z","shell.execute_reply":"2022-10-14T15:46:26.364407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(small_corpus)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.366510Z","iopub.execute_input":"2022-10-14T15:46:26.367465Z","iopub.status.idle":"2022-10-14T15:46:26.376244Z","shell.execute_reply.started":"2022-10-14T15:46:26.367431Z","shell.execute_reply":"2022-10-14T15:46:26.375249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"big_corpus = list(set(df['summary']))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.377475Z","iopub.execute_input":"2022-10-14T15:46:26.378453Z","iopub.status.idle":"2022-10-14T15:46:26.391020Z","shell.execute_reply.started":"2022-10-14T15:46:26.378417Z","shell.execute_reply":"2022-10-14T15:46:26.389790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(big_corpus)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.392664Z","iopub.execute_input":"2022-10-14T15:46:26.393207Z","iopub.status.idle":"2022-10-14T15:46:26.402571Z","shell.execute_reply.started":"2022-10-14T15:46:26.393170Z","shell.execute_reply":"2022-10-14T15:46:26.401587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install -U sentence-transformers\n!pip install faiss-gpu","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:26.405012Z","iopub.execute_input":"2022-10-14T15:46:26.405395Z","iopub.status.idle":"2022-10-14T15:46:54.941266Z","shell.execute_reply.started":"2022-10-14T15:46:26.405362Z","shell.execute_reply":"2022-10-14T15:46:54.939943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer, util\nimport time\nimport faiss","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:54.942898Z","iopub.execute_input":"2022-10-14T15:46:54.943336Z","iopub.status.idle":"2022-10-14T15:46:57.932004Z","shell.execute_reply.started":"2022-10-14T15:46:54.943290Z","shell.execute_reply":"2022-10-14T15:46:57.931067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n\nmodel = SentenceTransformer('all-distilroberta-v1')","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:46:57.934037Z","iopub.execute_input":"2022-10-14T15:46:57.935021Z","iopub.status.idle":"2022-10-14T15:59:41.648357Z","shell.execute_reply.started":"2022-10-14T15:46:57.934968Z","shell.execute_reply":"2022-10-14T15:59:41.646820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_embeddings = model.encode(small_corpus, convert_to_tensor=True, show_progress_bar=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:59:41.650350Z","iopub.execute_input":"2022-10-14T15:59:41.650728Z","iopub.status.idle":"2022-10-14T15:59:45.904264Z","shell.execute_reply.started":"2022-10-14T15:59:41.650686Z","shell.execute_reply":"2022-10-14T15:59:45.903282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"big_embeddings = model.encode(big_corpus, convert_to_tensor=True, show_progress_bar=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T15:59:45.905617Z","iopub.execute_input":"2022-10-14T15:59:45.906703Z","iopub.status.idle":"2022-10-14T16:00:09.715970Z","shell.execute_reply.started":"2022-10-14T15:59:45.906663Z","shell.execute_reply":"2022-10-14T16:00:09.715121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c_big_embeddings = model.encode(big_corpus, show_progress_bar=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:09.717452Z","iopub.execute_input":"2022-10-14T16:00:09.717914Z","iopub.status.idle":"2022-10-14T16:00:37.254680Z","shell.execute_reply.started":"2022-10-14T16:00:09.717875Z","shell.execute_reply":"2022-10-14T16:00:37.253647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c_small_embeddings = model.encode(small_corpus, show_progress_bar=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:37.263034Z","iopub.execute_input":"2022-10-14T16:00:37.263360Z","iopub.status.idle":"2022-10-14T16:00:37.680743Z","shell.execute_reply.started":"2022-10-14T16:00:37.263332Z","shell.execute_reply":"2022-10-14T16:00:37.679745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cosine_scores = util.cos_sim(c_small_embeddings, c_small_embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:37.682325Z","iopub.execute_input":"2022-10-14T16:00:37.682915Z","iopub.status.idle":"2022-10-14T16:00:37.737579Z","shell.execute_reply.started":"2022-10-14T16:00:37.682875Z","shell.execute_reply":"2022-10-14T16:00:37.736715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(cosine_scores)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:37.740056Z","iopub.execute_input":"2022-10-14T16:00:37.740699Z","iopub.status.idle":"2022-10-14T16:00:37.746870Z","shell.execute_reply.started":"2022-10-14T16:00:37.740664Z","shell.execute_reply":"2022-10-14T16:00:37.745863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_small_corpus = pd.DataFrame(small_corpus, columns = ['summary'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:37.748482Z","iopub.execute_input":"2022-10-14T16:00:37.748862Z","iopub.status.idle":"2022-10-14T16:00:37.756714Z","shell.execute_reply.started":"2022-10-14T16:00:37.748827Z","shell.execute_reply":"2022-10-14T16:00:37.755787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_big_corpus = pd.DataFrame(big_corpus, columns = ['summary'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:37.758189Z","iopub.execute_input":"2022-10-14T16:00:37.758618Z","iopub.status.idle":"2022-10-14T16:00:37.768608Z","shell.execute_reply.started":"2022-10-14T16:00:37.758583Z","shell.execute_reply":"2022-10-14T16:00:37.767657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged = pd.merge(df_small_corpus, df_filter, on=\"summary\", how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:37.770124Z","iopub.execute_input":"2022-10-14T16:00:37.770480Z","iopub.status.idle":"2022-10-14T16:00:37.790389Z","shell.execute_reply.started":"2022-10-14T16:00:37.770447Z","shell.execute_reply":"2022-10-14T16:00:37.789344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged_sel = df_merged[['summary', 'symbol']]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:37.793510Z","iopub.execute_input":"2022-10-14T16:00:37.793762Z","iopub.status.idle":"2022-10-14T16:00:37.802937Z","shell.execute_reply.started":"2022-10-14T16:00:37.793738Z","shell.execute_reply":"2022-10-14T16:00:37.801967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Визуализация","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:37.805971Z","iopub.execute_input":"2022-10-14T16:00:37.806277Z","iopub.status.idle":"2022-10-14T16:00:37.874961Z","shell.execute_reply.started":"2022-10-14T16:00:37.806253Z","shell.execute_reply":"2022-10-14T16:00:37.874097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.clustermap(np.asarray(cosine_scores), figsize=(20, 20), xticklabels=df_merged.symbol, yticklabels=df_merged.symbol)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:37.876921Z","iopub.execute_input":"2022-10-14T16:00:37.877558Z","iopub.status.idle":"2022-10-14T16:00:40.690927Z","shell.execute_reply.started":"2022-10-14T16:00:37.877511Z","shell.execute_reply":"2022-10-14T16:00:40.690043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"S = small_embeddings.cpu()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:40.691908Z","iopub.execute_input":"2022-10-14T16:00:40.692289Z","iopub.status.idle":"2022-10-14T16:00:40.698217Z","shell.execute_reply.started":"2022-10-14T16:00:40.692254Z","shell.execute_reply":"2022-10-14T16:00:40.697121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"B = big_embeddings.cpu()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:40.700006Z","iopub.execute_input":"2022-10-14T16:00:40.700862Z","iopub.status.idle":"2022-10-14T16:00:40.733906Z","shell.execute_reply.started":"2022-10-14T16:00:40.700826Z","shell.execute_reply":"2022-10-14T16:00:40.733117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install umap-learn","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:40.735392Z","iopub.execute_input":"2022-10-14T16:00:40.735808Z","iopub.status.idle":"2022-10-14T16:00:49.996757Z","shell.execute_reply.started":"2022-10-14T16:00:40.735774Z","shell.execute_reply":"2022-10-14T16:00:49.995536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import umap.umap_ as umap","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:00:49.998601Z","iopub.execute_input":"2022-10-14T16:00:49.999273Z","iopub.status.idle":"2022-10-14T16:01:06.482740Z","shell.execute_reply.started":"2022-10-14T16:00:49.999227Z","shell.execute_reply":"2022-10-14T16:01:06.481741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_umap_embedding = umap.UMAP(\n                                 n_neighbors=20,\n                                 min_dist=0,\n                                 n_components=2,\n                                 random_state=23,\n                                 repulsion_strength=1.0,).fit_transform(S)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:01:06.484285Z","iopub.execute_input":"2022-10-14T16:01:06.484653Z","iopub.status.idle":"2022-10-14T16:01:12.240309Z","shell.execute_reply.started":"2022-10-14T16:01:06.484614Z","shell.execute_reply":"2022-10-14T16:01:12.239294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"big_umap_embedding = umap.UMAP(\n                                 n_neighbors=20,\n                                 min_dist=0,\n                                 n_components=2,\n                                 random_state=23,\n                                 repulsion_strength=1.0,).fit_transform(B)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:01:12.241800Z","iopub.execute_input":"2022-10-14T16:01:12.242533Z","iopub.status.idle":"2022-10-14T16:01:36.865276Z","shell.execute_reply.started":"2022-10-14T16:01:12.242477Z","shell.execute_reply":"2022-10-14T16:01:36.864168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install hdbscan --no-build-isolation --no-binary :all:","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:01:36.867075Z","iopub.execute_input":"2022-10-14T16:01:36.867480Z","iopub.status.idle":"2022-10-14T16:02:47.950444Z","shell.execute_reply.started":"2022-10-14T16:01:36.867436Z","shell.execute_reply":"2022-10-14T16:02:47.949237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import hdbscan","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:47.952366Z","iopub.execute_input":"2022-10-14T16:02:47.954450Z","iopub.status.idle":"2022-10-14T16:02:47.974734Z","shell.execute_reply.started":"2022-10-14T16:02:47.954397Z","shell.execute_reply":"2022-10-14T16:02:47.973710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_labels = hdbscan.HDBSCAN(\nmin_samples=1,\nmin_cluster_size=3,\n).fit_predict(small_umap_embedding)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:47.976638Z","iopub.execute_input":"2022-10-14T16:02:47.977358Z","iopub.status.idle":"2022-10-14T16:02:47.988420Z","shell.execute_reply.started":"2022-10-14T16:02:47.977320Z","shell.execute_reply":"2022-10-14T16:02:47.987435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"big_labels = hdbscan.HDBSCAN(\nmin_samples=1,\nmin_cluster_size=10,\n).fit_predict(big_umap_embedding)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:47.990064Z","iopub.execute_input":"2022-10-14T16:02:47.991003Z","iopub.status.idle":"2022-10-14T16:02:48.178267Z","shell.execute_reply.started":"2022-10-14T16:02:47.990950Z","shell.execute_reply":"2022-10-14T16:02:48.177278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.pyplot import figure","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:48.179952Z","iopub.execute_input":"2022-10-14T16:02:48.180376Z","iopub.status.idle":"2022-10-14T16:02:48.186269Z","shell.execute_reply.started":"2022-10-14T16:02:48.180329Z","shell.execute_reply":"2022-10-14T16:02:48.185288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered = (small_labels >= 0)\n\nfigure(figsize=(15, 15), dpi=80)\n\nplt.scatter(small_umap_embedding[~clustered, 0],\n            small_umap_embedding[~clustered, 1],\n            c=(0.5, 0.5, 0.5),\n            s=10,\n            alpha=0.5)\nplt.scatter(small_umap_embedding[clustered, 0],\n            small_umap_embedding[clustered, 1],\n            c=(small_labels[clustered]),\n            s=20,\n            cmap='Spectral');\n\ntext = df_merged.symbol\n\nx = []\ny = []\nfor value in small_umap_embedding:\n    x.append(value[0])\n    y.append(value[1])\n    \nfor i in range(len(x)):    \n    plt.annotate(text[i],\n                  xy = (x[i], y[i]),\n                  xytext = (5, 2),\n                  textcoords = 'offset points',\n                  ha = 'right',\n                  va = 'bottom')","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:48.188136Z","iopub.execute_input":"2022-10-14T16:02:48.188728Z","iopub.status.idle":"2022-10-14T16:02:49.014635Z","shell.execute_reply.started":"2022-10-14T16:02:48.188692Z","shell.execute_reply":"2022-10-14T16:02:49.013450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered = (big_labels >= 0)\n\nfigure(figsize=(15, 15), dpi=80)\n\nplt.scatter(big_umap_embedding[~clustered, 0],\n            big_umap_embedding[~clustered, 1],\n            c=(0.5, 0.5, 0.5),\n            s=10,\n            alpha=0.5)\nplt.scatter(big_umap_embedding[clustered, 0],\n            big_umap_embedding[clustered, 1],\n            c=(big_labels[clustered]),\n            s=20,\n            cmap='Spectral');","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.016033Z","iopub.execute_input":"2022-10-14T16:02:49.016644Z","iopub.status.idle":"2022-10-14T16:02:49.569867Z","shell.execute_reply.started":"2022-10-14T16:02:49.016605Z","shell.execute_reply":"2022-10-14T16:02:49.569024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list = pd.DataFrame(np.column_stack([df_big_corpus, big_labels]), \n                               columns=['sentence', 'cluster'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.571474Z","iopub.execute_input":"2022-10-14T16:02:49.572153Z","iopub.status.idle":"2022-10-14T16:02:49.582175Z","shell.execute_reply.started":"2022-10-14T16:02:49.572110Z","shell.execute_reply":"2022-10-14T16:02:49.581042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_list = list(clustered_list['sentence'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.583656Z","iopub.execute_input":"2022-10-14T16:02:49.584151Z","iopub.status.idle":"2022-10-14T16:02:49.593178Z","shell.execute_reply.started":"2022-10-14T16:02:49.584045Z","shell.execute_reply":"2022-10-14T16:02:49.592204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_lookup = dict(zip(df['summary'], df['symbol']))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.594677Z","iopub.execute_input":"2022-10-14T16:02:49.595137Z","iopub.status.idle":"2022-10-14T16:02:49.607980Z","shell.execute_reply.started":"2022-10-14T16:02:49.595100Z","shell.execute_reply":"2022-10-14T16:02:49.607135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list['symbol'] = [dict_lookup[item] for item in key_list]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.609228Z","iopub.execute_input":"2022-10-14T16:02:49.609607Z","iopub.status.idle":"2022-10-14T16:02:49.621542Z","shell.execute_reply.started":"2022-10-14T16:02:49.609570Z","shell.execute_reply":"2022-10-14T16:02:49.620611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list['cluster'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.623068Z","iopub.execute_input":"2022-10-14T16:02:49.623486Z","iopub.status.idle":"2022-10-14T16:02:49.639409Z","shell.execute_reply.started":"2022-10-14T16:02:49.623453Z","shell.execute_reply":"2022-10-14T16:02:49.638553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list_filter_symbol = clustered_list.loc[clustered_list['symbol'] == 'TFRC']","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.640722Z","iopub.execute_input":"2022-10-14T16:02:49.640990Z","iopub.status.idle":"2022-10-14T16:02:49.651768Z","shell.execute_reply.started":"2022-10-14T16:02:49.640966Z","shell.execute_reply":"2022-10-14T16:02:49.650788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list_filter_symbol","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.653073Z","iopub.execute_input":"2022-10-14T16:02:49.654049Z","iopub.status.idle":"2022-10-14T16:02:49.666264Z","shell.execute_reply.started":"2022-10-14T16:02:49.654015Z","shell.execute_reply":"2022-10-14T16:02:49.665207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list_filter_group = clustered_list.loc[clustered_list['cluster'] == 173]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.667548Z","iopub.execute_input":"2022-10-14T16:02:49.667980Z","iopub.status.idle":"2022-10-14T16:02:49.678623Z","shell.execute_reply.started":"2022-10-14T16:02:49.667946Z","shell.execute_reply":"2022-10-14T16:02:49.677776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list_filter_group[\"sentence\"].replace('', np.nan, inplace=True)\n\nclustered_list_filter_group[\"sentence\"].replace('nan', np.nan, inplace=True)\n\nclustered_list_filter_group = clustered_list_filter_group[clustered_list_filter_group[\"sentence\"].notna()]\n\nclustered_list_filter_group = clustered_list_filter_group[clustered_list_filter_group[\"sentence\"].notnull()]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.679709Z","iopub.execute_input":"2022-10-14T16:02:49.679979Z","iopub.status.idle":"2022-10-14T16:02:49.694523Z","shell.execute_reply.started":"2022-10-14T16:02:49.679954Z","shell.execute_reply":"2022-10-14T16:02:49.693355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clustered_list_filter_group","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.696464Z","iopub.execute_input":"2022-10-14T16:02:49.696884Z","iopub.status.idle":"2022-10-14T16:02:49.711587Z","shell.execute_reply.started":"2022-10-14T16:02:49.696850Z","shell.execute_reply":"2022-10-14T16:02:49.710473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"def cluster_worldcloud(cluster_num):\n    comment_words = ''\n    stopwords = set(STOPWORDS)\n\n \n    for val in clustered_list[clustered_list['cluster']==cluster_num].sentence.values:\n      \n      \n        val = str(val)\n\n      \n        tokens = val.split()\n      \n      \n        for i in range(len(tokens)):\n            tokens[i] = tokens[i].lower()\n      \n        comment_words += \" \".join(tokens)+\" \"\n\n    wordcloud = WordCloud(width = 700, height = 700,\n                    background_color ='white',\n                    stopwords = stopwords,\n                    min_font_size = 10).generate(comment_words)\n\n\n                     \n    plt.figure(figsize = (10,5), facecolor = None)\n    plt.imshow(wordcloud)\n    plt.axis(\"off\")\n    plt.tight_layout(pad = 0)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.713294Z","iopub.execute_input":"2022-10-14T16:02:49.714025Z","iopub.status.idle":"2022-10-14T16:02:49.723303Z","shell.execute_reply.started":"2022-10-14T16:02:49.713991Z","shell.execute_reply":"2022-10-14T16:02:49.722294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport nltk\nfrom nltk import word_tokenize, sent_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import LancasterStemmer, WordNetLemmatizer\n\nfrom nltk.stem import WordNetLemmatizer\nnltk.download('wordnet')\nimport nltk\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nimport string\nstring.punctuation\nnltk.download('omw-1.4')\nfrom nltk.tokenize import TweetTokenizer\n\n%matplotlib inline\nsns.set()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:49.726418Z","iopub.execute_input":"2022-10-14T16:02:49.726674Z","iopub.status.idle":"2022-10-14T16:02:50.263979Z","shell.execute_reply.started":"2022-10-14T16:02:49.726650Z","shell.execute_reply":"2022-10-14T16:02:50.263057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cluster_worldcloud(173)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:50.265568Z","iopub.execute_input":"2022-10-14T16:02:50.265947Z","iopub.status.idle":"2022-10-14T16:02:51.012391Z","shell.execute_reply.started":"2022-10-14T16:02:50.265910Z","shell.execute_reply":"2022-10-14T16:02:51.011497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Получаем датафрейм ближайших соседей","metadata":{}},{"cell_type":"code","source":"pairs = []\nfor i in range(len(cosine_scores)-1):\n    for j in range(i+1, len(cosine_scores)):\n        pairs.append({'index': [i, j], 'score': cosine_scores[i][j]})","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.013872Z","iopub.execute_input":"2022-10-14T16:02:51.014420Z","iopub.status.idle":"2022-10-14T16:02:51.037018Z","shell.execute_reply.started":"2022-10-14T16:02:51.014384Z","shell.execute_reply":"2022-10-14T16:02:51.035923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Sort scores in decreasing order\npairs = sorted(pairs, key=lambda x: x['score'], reverse=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.038704Z","iopub.execute_input":"2022-10-14T16:02:51.039109Z","iopub.status.idle":"2022-10-14T16:02:51.154974Z","shell.execute_reply.started":"2022-10-14T16:02:51.039059Z","shell.execute_reply":"2022-10-14T16:02:51.154105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cos_neighbors = []\n\nfor pair in pairs:\n    i, j = pair['index']\n    cos_neighbors.append([small_corpus[i], small_corpus[j], np.asarray(pair['score'])])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.156728Z","iopub.execute_input":"2022-10-14T16:02:51.157145Z","iopub.status.idle":"2022-10-14T16:02:51.178602Z","shell.execute_reply.started":"2022-10-14T16:02:51.157095Z","shell.execute_reply":"2022-10-14T16:02:51.177482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flat_edgelist = [item for sublist in cos_neighbors for item in sublist]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.180005Z","iopub.execute_input":"2022-10-14T16:02:51.180425Z","iopub.status.idle":"2022-10-14T16:02:51.186657Z","shell.execute_reply.started":"2022-10-14T16:02:51.180386Z","shell.execute_reply":"2022-10-14T16:02:51.185622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"edgelist_3 = [list(flat_edgelist[i:i+3]) for i in range(0, len(flat_edgelist), 3)]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.188074Z","iopub.execute_input":"2022-10-14T16:02:51.189165Z","iopub.status.idle":"2022-10-14T16:02:51.198125Z","shell.execute_reply.started":"2022-10-14T16:02:51.189129Z","shell.execute_reply":"2022-10-14T16:02:51.197308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(edgelist_3)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.199581Z","iopub.execute_input":"2022-10-14T16:02:51.200187Z","iopub.status.idle":"2022-10-14T16:02:51.210993Z","shell.execute_reply.started":"2022-10-14T16:02:51.200152Z","shell.execute_reply":"2022-10-14T16:02:51.209718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors = pd.DataFrame(edgelist_3, columns = ['Id1', 'Id2', 'weight'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.212704Z","iopub.execute_input":"2022-10-14T16:02:51.213154Z","iopub.status.idle":"2022-10-14T16:02:51.221472Z","shell.execute_reply.started":"2022-10-14T16:02:51.213117Z","shell.execute_reply":"2022-10-14T16:02:51.220476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.222954Z","iopub.execute_input":"2022-10-14T16:02:51.223428Z","iopub.status.idle":"2022-10-14T16:02:51.236376Z","shell.execute_reply.started":"2022-10-14T16:02:51.223393Z","shell.execute_reply":"2022-10-14T16:02:51.235238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_list = list(df_neighbors['Id1'])\nkey_list_2 = list(df_neighbors['Id2'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.238206Z","iopub.execute_input":"2022-10-14T16:02:51.238620Z","iopub.status.idle":"2022-10-14T16:02:51.246580Z","shell.execute_reply.started":"2022-10-14T16:02:51.238586Z","shell.execute_reply":"2022-10-14T16:02:51.245599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_lookup = dict(zip(df_filter['summary'], df_filter['symbol']))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.248154Z","iopub.execute_input":"2022-10-14T16:02:51.248487Z","iopub.status.idle":"2022-10-14T16:02:51.254659Z","shell.execute_reply.started":"2022-10-14T16:02:51.248453Z","shell.execute_reply":"2022-10-14T16:02:51.253663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors['Id1'] = [dict_lookup[item] for item in key_list]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.255989Z","iopub.execute_input":"2022-10-14T16:02:51.256660Z","iopub.status.idle":"2022-10-14T16:02:51.264873Z","shell.execute_reply.started":"2022-10-14T16:02:51.256621Z","shell.execute_reply":"2022-10-14T16:02:51.264230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors['Id2'] = [dict_lookup[item] for item in key_list_2]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.266938Z","iopub.execute_input":"2022-10-14T16:02:51.267364Z","iopub.status.idle":"2022-10-14T16:02:51.275744Z","shell.execute_reply.started":"2022-10-14T16:02:51.267330Z","shell.execute_reply":"2022-10-14T16:02:51.274833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors = df_neighbors.sort_values(by=['weight'], ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.277074Z","iopub.execute_input":"2022-10-14T16:02:51.277492Z","iopub.status.idle":"2022-10-14T16:02:51.305064Z","shell.execute_reply.started":"2022-10-14T16:02:51.277458Z","shell.execute_reply":"2022-10-14T16:02:51.304135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.306521Z","iopub.execute_input":"2022-10-14T16:02:51.306899Z","iopub.status.idle":"2022-10-14T16:02:51.321098Z","shell.execute_reply.started":"2022-10-14T16:02:51.306866Z","shell.execute_reply":"2022-10-14T16:02:51.320223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.324461Z","iopub.execute_input":"2022-10-14T16:02:51.324821Z","iopub.status.idle":"2022-10-14T16:02:51.333206Z","shell.execute_reply.started":"2022-10-14T16:02:51.324796Z","shell.execute_reply":"2022-10-14T16:02:51.332127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors_filter = df_neighbors.loc[df_neighbors['Id1'] == 'TNFRSF9']","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.335131Z","iopub.execute_input":"2022-10-14T16:02:51.335652Z","iopub.status.idle":"2022-10-14T16:02:51.343876Z","shell.execute_reply.started":"2022-10-14T16:02:51.335617Z","shell.execute_reply":"2022-10-14T16:02:51.343007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors_filter.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.345296Z","iopub.execute_input":"2022-10-14T16:02:51.345924Z","iopub.status.idle":"2022-10-14T16:02:51.360269Z","shell.execute_reply.started":"2022-10-14T16:02:51.345889Z","shell.execute_reply":"2022-10-14T16:02:51.359236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Строим граф","metadata":{}},{"cell_type":"code","source":"import networkx as nx","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.361664Z","iopub.execute_input":"2022-10-14T16:02:51.362174Z","iopub.status.idle":"2022-10-14T16:02:51.445788Z","shell.execute_reply.started":"2022-10-14T16:02:51.362139Z","shell.execute_reply":"2022-10-14T16:02:51.444888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors_filter = df_neighbors[df_neighbors['weight'] > 0.6]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.447273Z","iopub.execute_input":"2022-10-14T16:02:51.447641Z","iopub.status.idle":"2022-10-14T16:02:51.457798Z","shell.execute_reply.started":"2022-10-14T16:02:51.447605Z","shell.execute_reply":"2022-10-14T16:02:51.456798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"G  = nx.from_pandas_edgelist(\n    df_neighbors_filter,\n    source='Id1',\n    target='Id2',\n    edge_attr='weight'\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.474443Z","iopub.execute_input":"2022-10-14T16:02:51.475274Z","iopub.status.idle":"2022-10-14T16:02:51.482701Z","shell.execute_reply.started":"2022-10-14T16:02:51.475240Z","shell.execute_reply":"2022-10-14T16:02:51.481703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(nx.info(G))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.484306Z","iopub.execute_input":"2022-10-14T16:02:51.484737Z","iopub.status.idle":"2022-10-14T16:02:51.494521Z","shell.execute_reply.started":"2022-10-14T16:02:51.484702Z","shell.execute_reply":"2022-10-14T16:02:51.493105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from community import community_louvain","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.496244Z","iopub.execute_input":"2022-10-14T16:02:51.496604Z","iopub.status.idle":"2022-10-14T16:02:51.507268Z","shell.execute_reply.started":"2022-10-14T16:02:51.496570Z","shell.execute_reply":"2022-10-14T16:02:51.506215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"communities =community_louvain.best_partition(G)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.508878Z","iopub.execute_input":"2022-10-14T16:02:51.509297Z","iopub.status.idle":"2022-10-14T16:02:51.555660Z","shell.execute_reply.started":"2022-10-14T16:02:51.509262Z","shell.execute_reply":"2022-10-14T16:02:51.554802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"community_id = [communities[node] for node in G.nodes()]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.556879Z","iopub.execute_input":"2022-10-14T16:02:51.557205Z","iopub.status.idle":"2022-10-14T16:02:51.564160Z","shell.execute_reply.started":"2022-10-14T16:02:51.557178Z","shell.execute_reply":"2022-10-14T16:02:51.563247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pos = nx.spring_layout(G)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.565831Z","iopub.execute_input":"2022-10-14T16:02:51.566375Z","iopub.status.idle":"2022-10-14T16:02:51.600677Z","shell.execute_reply.started":"2022-10-14T16:02:51.566339Z","shell.execute_reply":"2022-10-14T16:02:51.599835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install hvplot","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:02:51.602051Z","iopub.execute_input":"2022-10-14T16:02:51.602427Z","iopub.status.idle":"2022-10-14T16:03:02.509107Z","shell.execute_reply.started":"2022-10-14T16:02:51.602393Z","shell.execute_reply":"2022-10-14T16:03:02.507864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import hvplot.networkx as hvnx\nimport holoviews as hv\nfrom bokeh.models import HoverTool\nhv.extension('bokeh')","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:02.510714Z","iopub.execute_input":"2022-10-14T16:03:02.511044Z","iopub.status.idle":"2022-10-14T16:03:06.200758Z","shell.execute_reply.started":"2022-10-14T16:03:02.511010Z","shell.execute_reply":"2022-10-14T16:03:06.199720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hvnx.draw(G, pos, node_color=community_id, \n          edge_color='lightgray').opts(tools=[HoverTool(tooltips=[('index', '@index_hover')])])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.202316Z","iopub.execute_input":"2022-10-14T16:03:06.203017Z","iopub.status.idle":"2022-10-14T16:03:06.417348Z","shell.execute_reply.started":"2022-10-14T16:03:06.202977Z","shell.execute_reply":"2022-10-14T16:03:06.416296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nx.is_connected(G)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.419006Z","iopub.execute_input":"2022-10-14T16:03:06.419475Z","iopub.status.idle":"2022-10-14T16:03:06.427170Z","shell.execute_reply.started":"2022-10-14T16:03:06.419422Z","shell.execute_reply":"2022-10-14T16:03:06.426158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nx.number_connected_components(G)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.428784Z","iopub.execute_input":"2022-10-14T16:03:06.429491Z","iopub.status.idle":"2022-10-14T16:03:06.440276Z","shell.execute_reply.started":"2022-10-14T16:03:06.429444Z","shell.execute_reply":"2022-10-14T16:03:06.439248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cliques = list(nx.find_cliques(G))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.441504Z","iopub.execute_input":"2022-10-14T16:03:06.442242Z","iopub.status.idle":"2022-10-14T16:03:06.452285Z","shell.execute_reply.started":"2022-10-14T16:03:06.442195Z","shell.execute_reply":"2022-10-14T16:03:06.451241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clique_number = len(list(cliques))\nprint(clique_number)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.453776Z","iopub.execute_input":"2022-10-14T16:03:06.454143Z","iopub.status.idle":"2022-10-14T16:03:06.463573Z","shell.execute_reply.started":"2022-10-14T16:03:06.454103Z","shell.execute_reply":"2022-10-14T16:03:06.462561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_cliques = sorted(list(nx.find_cliques(G)),key=len)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.465303Z","iopub.execute_input":"2022-10-14T16:03:06.465669Z","iopub.status.idle":"2022-10-14T16:03:06.476233Z","shell.execute_reply.started":"2022-10-14T16:03:06.465629Z","shell.execute_reply":"2022-10-14T16:03:06.475291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_clique_nodes = set()\n\nfor nodelist in sorted_cliques[-4:-1]:\n    for node in nodelist:\n        max_clique_nodes.add(node)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.477780Z","iopub.execute_input":"2022-10-14T16:03:06.478160Z","iopub.status.idle":"2022-10-14T16:03:06.486025Z","shell.execute_reply.started":"2022-10-14T16:03:06.478125Z","shell.execute_reply":"2022-10-14T16:03:06.485146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_clique = G.subgraph(max_clique_nodes)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.487640Z","iopub.execute_input":"2022-10-14T16:03:06.488017Z","iopub.status.idle":"2022-10-14T16:03:06.498989Z","shell.execute_reply.started":"2022-10-14T16:03:06.487979Z","shell.execute_reply":"2022-10-14T16:03:06.498236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(nx.info(max_clique))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.500130Z","iopub.execute_input":"2022-10-14T16:03:06.501007Z","iopub.status.idle":"2022-10-14T16:03:06.512074Z","shell.execute_reply.started":"2022-10-14T16:03:06.500973Z","shell.execute_reply":"2022-10-14T16:03:06.511038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_clique_communities =community_louvain.best_partition(max_clique)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.513201Z","iopub.execute_input":"2022-10-14T16:03:06.514077Z","iopub.status.idle":"2022-10-14T16:03:06.527961Z","shell.execute_reply.started":"2022-10-14T16:03:06.514044Z","shell.execute_reply":"2022-10-14T16:03:06.527063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_clique_community_id = [max_clique_communities[node] for node in max_clique.nodes()]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.529437Z","iopub.execute_input":"2022-10-14T16:03:06.530098Z","iopub.status.idle":"2022-10-14T16:03:06.538926Z","shell.execute_reply.started":"2022-10-14T16:03:06.530047Z","shell.execute_reply":"2022-10-14T16:03:06.537909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_clique_pos = nx.spring_layout(max_clique)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.540329Z","iopub.execute_input":"2022-10-14T16:03:06.541112Z","iopub.status.idle":"2022-10-14T16:03:06.557110Z","shell.execute_reply.started":"2022-10-14T16:03:06.541060Z","shell.execute_reply":"2022-10-14T16:03:06.556191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hvnx.draw(max_clique, max_clique_pos, node_color=max_clique_community_id, \n          edge_color='lightgray').opts(tools=[HoverTool(tooltips=[('index', '@index_hover')])])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.558430Z","iopub.execute_input":"2022-10-14T16:03:06.558921Z","iopub.status.idle":"2022-10-14T16:03:06.712512Z","shell.execute_reply.started":"2022-10-14T16:03:06.558887Z","shell.execute_reply":"2022-10-14T16:03:06.711479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"community_list = pd.DataFrame(np.column_stack([G.nodes, community_id]), \n                               columns=['node', 'community'])\n\ncommunity_list.community = community_list.community.astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.714188Z","iopub.execute_input":"2022-10-14T16:03:06.714549Z","iopub.status.idle":"2022-10-14T16:03:06.721136Z","shell.execute_reply.started":"2022-10-14T16:03:06.714513Z","shell.execute_reply":"2022-10-14T16:03:06.719967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"community_list.community.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.722665Z","iopub.execute_input":"2022-10-14T16:03:06.723361Z","iopub.status.idle":"2022-10-14T16:03:06.735599Z","shell.execute_reply.started":"2022-10-14T16:03:06.723326Z","shell.execute_reply":"2022-10-14T16:03:06.734554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"community_list_filter = community_list[community_list['community'] == 0]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.737393Z","iopub.execute_input":"2022-10-14T16:03:06.737742Z","iopub.status.idle":"2022-10-14T16:03:06.746018Z","shell.execute_reply.started":"2022-10-14T16:03:06.737708Z","shell.execute_reply":"2022-10-14T16:03:06.745070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"community_list_filter","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.747700Z","iopub.execute_input":"2022-10-14T16:03:06.748052Z","iopub.status.idle":"2022-10-14T16:03:06.760417Z","shell.execute_reply.started":"2022-10-14T16:03:06.748009Z","shell.execute_reply":"2022-10-14T16:03:06.759394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neighbors_list = []\nfor n in G.nodes:\n    neighbors_list.append([n, sorted(G[n].items(), key=lambda edge: edge[1]['weight'])])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.762261Z","iopub.execute_input":"2022-10-14T16:03:06.762728Z","iopub.status.idle":"2022-10-14T16:03:06.771732Z","shell.execute_reply.started":"2022-10-14T16:03:06.762693Z","shell.execute_reply":"2022-10-14T16:03:06.770881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graph_neighbors = (pd.DataFrame(neighbors_list, columns=['node', 'neighbor'])\n      .explode('neighbor', ignore_index=True))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.773343Z","iopub.execute_input":"2022-10-14T16:03:06.773765Z","iopub.status.idle":"2022-10-14T16:03:06.787571Z","shell.execute_reply.started":"2022-10-14T16:03:06.773731Z","shell.execute_reply":"2022-10-14T16:03:06.786606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"graph_neighbors.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.789188Z","iopub.execute_input":"2022-10-14T16:03:06.789633Z","iopub.status.idle":"2022-10-14T16:03:06.806440Z","shell.execute_reply.started":"2022-10-14T16:03:06.789598Z","shell.execute_reply":"2022-10-14T16:03:06.805454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pr_summary(G):\n    pr = nx.pagerank(G)\n    df = pd.DataFrame.from_dict({\n        'node': list(pr.keys()),\n        'centrality_pr': list(pr.values())\n    })\n    return df.sort_values('centrality_pr', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.808735Z","iopub.execute_input":"2022-10-14T16:03:06.808990Z","iopub.status.idle":"2022-10-14T16:03:06.815656Z","shell.execute_reply.started":"2022-10-14T16:03:06.808966Z","shell.execute_reply":"2022-10-14T16:03:06.814659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pr = pd.DataFrame(pr_summary(G), columns=['node', 'centrality_pr'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.817280Z","iopub.execute_input":"2022-10-14T16:03:06.817944Z","iopub.status.idle":"2022-10-14T16:03:06.876157Z","shell.execute_reply.started":"2022-10-14T16:03:06.817909Z","shell.execute_reply":"2022-10-14T16:03:06.875347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pr.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.877645Z","iopub.execute_input":"2022-10-14T16:03:06.877991Z","iopub.status.idle":"2022-10-14T16:03:06.889935Z","shell.execute_reply.started":"2022-10-14T16:03:06.877956Z","shell.execute_reply":"2022-10-14T16:03:06.888749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def bc_summary(G):\n    bc = nx.betweenness_centrality(G)\n    df = pd.DataFrame.from_dict({\n        'node': list(bc.keys()),\n        'centrality_bc': list(bc.values())\n    })\n    return df.sort_values('centrality_bc', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.891486Z","iopub.execute_input":"2022-10-14T16:03:06.892392Z","iopub.status.idle":"2022-10-14T16:03:06.898691Z","shell.execute_reply.started":"2022-10-14T16:03:06.892355Z","shell.execute_reply":"2022-10-14T16:03:06.897434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bc = pd.DataFrame(bc_summary(G), columns=['node', 'centrality_bc'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.900217Z","iopub.execute_input":"2022-10-14T16:03:06.900637Z","iopub.status.idle":"2022-10-14T16:03:06.954676Z","shell.execute_reply.started":"2022-10-14T16:03:06.900603Z","shell.execute_reply":"2022-10-14T16:03:06.953855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bc.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.955788Z","iopub.execute_input":"2022-10-14T16:03:06.956140Z","iopub.status.idle":"2022-10-14T16:03:06.967469Z","shell.execute_reply.started":"2022-10-14T16:03:06.956098Z","shell.execute_reply":"2022-10-14T16:03:06.966453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import reduce","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.968747Z","iopub.execute_input":"2022-10-14T16:03:06.969454Z","iopub.status.idle":"2022-10-14T16:03:06.977208Z","shell.execute_reply.started":"2022-10-14T16:03:06.969416Z","shell.execute_reply":"2022-10-14T16:03:06.976210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [community_list, bc, pr]  \ndf_sum = reduce(lambda left,right: pd.merge(left,right,on='node'), dfs)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.978514Z","iopub.execute_input":"2022-10-14T16:03:06.979023Z","iopub.status.idle":"2022-10-14T16:03:06.994830Z","shell.execute_reply.started":"2022-10-14T16:03:06.978987Z","shell.execute_reply":"2022-10-14T16:03:06.993789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sum = df_sum[(df_sum['centrality_pr'] > 0) & (df_sum['centrality_bc'] > 0)]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:06.996096Z","iopub.execute_input":"2022-10-14T16:03:06.996446Z","iopub.status.idle":"2022-10-14T16:03:07.017120Z","shell.execute_reply.started":"2022-10-14T16:03:06.996408Z","shell.execute_reply":"2022-10-14T16:03:07.013850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import bokeh\nfrom bokeh.models import ColumnDataSource, Label, LabelSet, Range1d\nfrom bokeh.plotting import figure, output_file, show\nimport bokeh.plotting as bpl\nfrom bokeh.models import LinearColorMapper\nbokeh.io.output_notebook()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:07.018803Z","iopub.execute_input":"2022-10-14T16:03:07.019654Z","iopub.status.idle":"2022-10-14T16:03:07.034355Z","shell.execute_reply.started":"2022-10-14T16:03:07.019618Z","shell.execute_reply":"2022-10-14T16:03:07.033405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"source = ColumnDataSource(pd.DataFrame(df_sum))\n\np = figure(title='centralities map')\n\ncommunity = list(df_sum['community'].unique())\n\nexp_cmap = LinearColorMapper(palette=\"Turbo256\", \n                             low = min(community), \n                             high = max(community))\n\n\n\np.scatter(x='centrality_bc', y='centrality_pr', \n          line_color=None,\n          fill_color={\"field\":\"community\", \"transform\":exp_cmap}, \n          size=8, source=source)\n\n\np.xaxis[0].axis_label = 'centrality_bc'\np.xaxis.major_label_text_color = \"green\"\n\n\np.yaxis[0].axis_label = 'centrality_pr'\np.yaxis.major_label_text_color = \"orange\"\np.yaxis.major_label_orientation = \"vertical\"\n\n\nx1 = [df_sum['centrality_bc'].min(), df_sum['centrality_bc'].max()]\ny1 = [df_sum['centrality_pr'].min(), df_sum['centrality_pr'].max()]\n\np.line(x1, y1, line_width=1,color='red', line_dash='dashed')\n\nlabels = LabelSet(x='centrality_bc', y='centrality_pr', text='node',\n              x_offset=5, y_offset=5, source=source, render_mode='canvas')\n\np.add_layout(labels)\n\nfrom bokeh.models import ColorBar\nbar = ColorBar(color_mapper=exp_cmap, location=(0,0))\np.add_layout(bar, \"left\")\n\n\nshow(p)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:07.035855Z","iopub.execute_input":"2022-10-14T16:03:07.036436Z","iopub.status.idle":"2022-10-14T16:03:07.117898Z","shell.execute_reply.started":"2022-10-14T16:03:07.036401Z","shell.execute_reply":"2022-10-14T16:03:07.116995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Смотрим датафрейм ближайших соседей для проверки корреляции ","metadata":{}},{"cell_type":"markdown","source":"Корреляции генов и рнк получены с ипсользованием кода [этого](https://www.kaggle.com/code/alexandervc/mmscel-eda-targets-citeseq-02) блокнота.","metadata":{}},{"cell_type":"code","source":"!pip install faiss-gpu","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:07.119420Z","iopub.execute_input":"2022-10-14T16:03:07.119782Z","iopub.status.idle":"2022-10-14T16:03:16.412443Z","shell.execute_reply.started":"2022-10-14T16:03:07.119746Z","shell.execute_reply":"2022-10-14T16:03:16.411055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import faiss","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:16.416505Z","iopub.execute_input":"2022-10-14T16:03:16.416841Z","iopub.status.idle":"2022-10-14T16:03:16.421624Z","shell.execute_reply.started":"2022-10-14T16:03:16.416806Z","shell.execute_reply":"2022-10-14T16:03:16.420620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"faiss.normalize_L2(c_big_embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:16.422978Z","iopub.execute_input":"2022-10-14T16:03:16.423338Z","iopub.status.idle":"2022-10-14T16:03:16.445933Z","shell.execute_reply.started":"2022-10-14T16:03:16.423303Z","shell.execute_reply":"2022-10-14T16:03:16.445115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"d = c_big_embeddings.shape[1]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:16.448986Z","iopub.execute_input":"2022-10-14T16:03:16.449276Z","iopub.status.idle":"2022-10-14T16:03:16.456113Z","shell.execute_reply.started":"2022-10-14T16:03:16.449252Z","shell.execute_reply":"2022-10-14T16:03:16.455111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = faiss.IndexFlatL2(d)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:16.459755Z","iopub.execute_input":"2022-10-14T16:03:16.460047Z","iopub.status.idle":"2022-10-14T16:03:16.466246Z","shell.execute_reply.started":"2022-10-14T16:03:16.460022Z","shell.execute_reply":"2022-10-14T16:03:16.465285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index.add(c_big_embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:16.467445Z","iopub.execute_input":"2022-10-14T16:03:16.468042Z","iopub.status.idle":"2022-10-14T16:03:16.489170Z","shell.execute_reply.started":"2022-10-14T16:03:16.468007Z","shell.execute_reply":"2022-10-14T16:03:16.488252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index.ntotal","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:16.492099Z","iopub.execute_input":"2022-10-14T16:03:16.492354Z","iopub.status.idle":"2022-10-14T16:03:16.501179Z","shell.execute_reply.started":"2022-10-14T16:03:16.492330Z","shell.execute_reply":"2022-10-14T16:03:16.500253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def search(query):\n    query_vector = model.encode([query])\n    faiss.normalize_L2(query_vector)\n    k = 100\n    similarities, similarities_ids = index.search(query_vector.astype(np.float32), k)\n    similarities = np.around(np.clip(similarities, 0, 1), decimals=4)\n    highest_val_idx = np.argmax([sum(sim_vec) for sim_vec in similarities])\n    l1 = [big_corpus[_id] for _id in similarities_ids.tolist()[0]]\n    l2 = similarities[highest_val_idx].tolist()\n    return list(zip(l1,l2))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:16.502851Z","iopub.execute_input":"2022-10-14T16:03:16.503248Z","iopub.status.idle":"2022-10-14T16:03:16.511506Z","shell.execute_reply.started":"2022-10-14T16:03:16.503169Z","shell.execute_reply":"2022-10-14T16:03:16.510449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\nedgelist = []\nfor word in tqdm(big_corpus):\n    edges = []\n    for s, w in search(word):\n        edges.extend((word, s, w))\n    edgelist.append(edges)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:03:16.513304Z","iopub.execute_input":"2022-10-14T16:03:16.513765Z","iopub.status.idle":"2022-10-14T16:12:20.134978Z","shell.execute_reply.started":"2022-10-14T16:03:16.513680Z","shell.execute_reply":"2022-10-14T16:12:20.133895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"flat_edgelist = [item for sublist in edgelist for item in sublist]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:20.136553Z","iopub.execute_input":"2022-10-14T16:12:20.136925Z","iopub.status.idle":"2022-10-14T16:12:20.289748Z","shell.execute_reply.started":"2022-10-14T16:12:20.136889Z","shell.execute_reply":"2022-10-14T16:12:20.288815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"edgelist_3 = [list(flat_edgelist[i:i+3]) for i in range(0, len(flat_edgelist), 3)]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:20.291326Z","iopub.execute_input":"2022-10-14T16:12:20.291709Z","iopub.status.idle":"2022-10-14T16:12:22.172577Z","shell.execute_reply.started":"2022-10-14T16:12:20.291672Z","shell.execute_reply":"2022-10-14T16:12:22.171581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors = pd.DataFrame(edgelist_3, columns = ['Id1', 'Id2', 'weight'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:22.173989Z","iopub.execute_input":"2022-10-14T16:12:22.174610Z","iopub.status.idle":"2022-10-14T16:12:22.459616Z","shell.execute_reply.started":"2022-10-14T16:12:22.174567Z","shell.execute_reply":"2022-10-14T16:12:22.458597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_list = list(df_neighbors['Id1'])\nkey_list_2 = list(df_neighbors['Id2'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:22.460991Z","iopub.execute_input":"2022-10-14T16:12:22.461411Z","iopub.status.idle":"2022-10-14T16:12:22.621829Z","shell.execute_reply.started":"2022-10-14T16:12:22.461368Z","shell.execute_reply":"2022-10-14T16:12:22.620693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_lookup = dict(zip(df['summary'], df['symbol']))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:22.623447Z","iopub.execute_input":"2022-10-14T16:12:22.623843Z","iopub.status.idle":"2022-10-14T16:12:22.638041Z","shell.execute_reply.started":"2022-10-14T16:12:22.623806Z","shell.execute_reply":"2022-10-14T16:12:22.637023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors['Id1'] = [dict_lookup[item] for item in key_list]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:22.639534Z","iopub.execute_input":"2022-10-14T16:12:22.639891Z","iopub.status.idle":"2022-10-14T16:12:22.766685Z","shell.execute_reply.started":"2022-10-14T16:12:22.639857Z","shell.execute_reply":"2022-10-14T16:12:22.765699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors['Id2'] = [dict_lookup[item] for item in key_list_2]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:22.767851Z","iopub.execute_input":"2022-10-14T16:12:22.768241Z","iopub.status.idle":"2022-10-14T16:12:22.928360Z","shell.execute_reply.started":"2022-10-14T16:12:22.768202Z","shell.execute_reply":"2022-10-14T16:12:22.927379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:22.929773Z","iopub.execute_input":"2022-10-14T16:12:22.930167Z","iopub.status.idle":"2022-10-14T16:12:22.943641Z","shell.execute_reply.started":"2022-10-14T16:12:22.930122Z","shell.execute_reply":"2022-10-14T16:12:22.942534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors = df_neighbors.sort_values('weight' , ascending = False)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:22.944889Z","iopub.execute_input":"2022-10-14T16:12:22.945936Z","iopub.status.idle":"2022-10-14T16:12:23.119288Z","shell.execute_reply.started":"2022-10-14T16:12:22.945900Z","shell.execute_reply":"2022-10-14T16:12:23.118127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:23.120673Z","iopub.execute_input":"2022-10-14T16:12:23.121582Z","iopub.status.idle":"2022-10-14T16:12:23.128600Z","shell.execute_reply.started":"2022-10-14T16:12:23.121533Z","shell.execute_reply":"2022-10-14T16:12:23.127658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"group_df_neighbors = df_neighbors.groupby('Id1')\n\nneighbors_list = group_df_neighbors.apply(lambda x: x['Id2'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:23.129905Z","iopub.execute_input":"2022-10-14T16:12:23.130587Z","iopub.status.idle":"2022-10-14T16:12:24.253453Z","shell.execute_reply.started":"2022-10-14T16:12:23.130548Z","shell.execute_reply":"2022-10-14T16:12:24.252445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors_list = pd.DataFrame(neighbors_list, columns = ['neighbors'])","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:24.255039Z","iopub.execute_input":"2022-10-14T16:12:24.255495Z","iopub.status.idle":"2022-10-14T16:12:24.261464Z","shell.execute_reply.started":"2022-10-14T16:12:24.255456Z","shell.execute_reply":"2022-10-14T16:12:24.259969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors_filter = df_neighbors_list[df_neighbors_list.index == 'CD44']","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:24.263261Z","iopub.execute_input":"2022-10-14T16:12:24.263770Z","iopub.status.idle":"2022-10-14T16:12:24.272749Z","shell.execute_reply.started":"2022-10-14T16:12:24.263732Z","shell.execute_reply":"2022-10-14T16:12:24.271790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors_filter","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:24.273877Z","iopub.execute_input":"2022-10-14T16:12:24.274254Z","iopub.status.idle":"2022-10-14T16:12:24.289999Z","shell.execute_reply.started":"2022-10-14T16:12:24.274213Z","shell.execute_reply":"2022-10-14T16:12:24.288998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sel_list = df_neighbors_filter.neighbors.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:24.291539Z","iopub.execute_input":"2022-10-14T16:12:24.292207Z","iopub.status.idle":"2022-10-14T16:12:24.298969Z","shell.execute_reply.started":"2022-10-14T16:12:24.292171Z","shell.execute_reply":"2022-10-14T16:12:24.298110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sel_list","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:24.300645Z","iopub.execute_input":"2022-10-14T16:12:24.300989Z","iopub.status.idle":"2022-10-14T16:12:24.312811Z","shell.execute_reply.started":"2022-10-14T16:12:24.300956Z","shell.execute_reply":"2022-10-14T16:12:24.311853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ищем соседей для CD-белков в датасете protein-protein interactions","metadata":{}},{"cell_type":"code","source":"fn = '/kaggle/input/protein-protein-interactions/BIOGRID-ALL-4.3.195.tab3/BIOGRID-ALL-4.3.195.tab3.txt'\ndf_ppi = pd.read_csv(fn, sep ='\\t')","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:24.315020Z","iopub.execute_input":"2022-10-14T16:12:24.315322Z","iopub.status.idle":"2022-10-14T16:12:46.862186Z","shell.execute_reply.started":"2022-10-14T16:12:24.315297Z","shell.execute_reply":"2022-10-14T16:12:46.861211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:46.863593Z","iopub.execute_input":"2022-10-14T16:12:46.864121Z","iopub.status.idle":"2022-10-14T16:12:46.891640Z","shell.execute_reply.started":"2022-10-14T16:12:46.864057Z","shell.execute_reply":"2022-10-14T16:12:46.890760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:14:19.637933Z","iopub.execute_input":"2022-10-14T16:14:19.638932Z","iopub.status.idle":"2022-10-14T16:14:19.647803Z","shell.execute_reply.started":"2022-10-14T16:14:19.638890Z","shell.execute_reply":"2022-10-14T16:14:19.646419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:46.892879Z","iopub.execute_input":"2022-10-14T16:12:46.893365Z","iopub.status.idle":"2022-10-14T16:12:46.903710Z","shell.execute_reply.started":"2022-10-14T16:12:46.893327Z","shell.execute_reply":"2022-10-14T16:12:46.902814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"genes_cd_list = list(set(df_filter['symbol']))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:46.905116Z","iopub.execute_input":"2022-10-14T16:12:46.905734Z","iopub.status.idle":"2022-10-14T16:12:46.914203Z","shell.execute_reply.started":"2022-10-14T16:12:46.905700Z","shell.execute_reply":"2022-10-14T16:12:46.913258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors_filter = df_neighbors[(df_neighbors['Id1'].isin(genes_cd_list)) | (df_neighbors['Id2'].isin(genes_cd_list))]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:46.915505Z","iopub.execute_input":"2022-10-14T16:12:46.915956Z","iopub.status.idle":"2022-10-14T16:12:47.088684Z","shell.execute_reply.started":"2022-10-14T16:12:46.915878Z","shell.execute_reply":"2022-10-14T16:12:47.087441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_neighbors_filter.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:47.089957Z","iopub.execute_input":"2022-10-14T16:12:47.091704Z","iopub.status.idle":"2022-10-14T16:12:47.105037Z","shell.execute_reply.started":"2022-10-14T16:12:47.091666Z","shell.execute_reply":"2022-10-14T16:12:47.103443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"genes_neighbors_list_1 = list(set(df_neighbors_filter['Id1']))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:47.106710Z","iopub.execute_input":"2022-10-14T16:12:47.107074Z","iopub.status.idle":"2022-10-14T16:12:47.116137Z","shell.execute_reply.started":"2022-10-14T16:12:47.107036Z","shell.execute_reply":"2022-10-14T16:12:47.115009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"genes_neighbors_list_2 = list(set(df_neighbors_filter['Id2']))","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:47.117723Z","iopub.execute_input":"2022-10-14T16:12:47.118075Z","iopub.status.idle":"2022-10-14T16:12:47.127886Z","shell.execute_reply.started":"2022-10-14T16:12:47.118041Z","shell.execute_reply":"2022-10-14T16:12:47.126953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"genes_neighbors_list = genes_neighbors_list_1 + genes_neighbors_list_2","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:47.129439Z","iopub.execute_input":"2022-10-14T16:12:47.129804Z","iopub.status.idle":"2022-10-14T16:12:47.137930Z","shell.execute_reply.started":"2022-10-14T16:12:47.129768Z","shell.execute_reply":"2022-10-14T16:12:47.136998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(genes_neighbors_list)","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:12:47.140732Z","iopub.execute_input":"2022-10-14T16:12:47.141236Z","iopub.status.idle":"2022-10-14T16:12:47.150154Z","shell.execute_reply.started":"2022-10-14T16:12:47.141199Z","shell.execute_reply":"2022-10-14T16:12:47.149141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_filter = df_ppi[(df_ppi['Official Symbol Interactor A'].isin(genes_neighbors_list)) & (df_ppi['Official Symbol Interactor B'].isin(genes_neighbors_list))]","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:20:42.281228Z","iopub.execute_input":"2022-10-14T16:20:42.281844Z","iopub.status.idle":"2022-10-14T16:20:42.525093Z","shell.execute_reply.started":"2022-10-14T16:20:42.281806Z","shell.execute_reply":"2022-10-14T16:20:42.524027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_filter.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:20:44.035748Z","iopub.execute_input":"2022-10-14T16:20:44.036432Z","iopub.status.idle":"2022-10-14T16:20:44.063977Z","shell.execute_reply.started":"2022-10-14T16:20:44.036390Z","shell.execute_reply":"2022-10-14T16:20:44.062820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_filter.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:20:47.106572Z","iopub.execute_input":"2022-10-14T16:20:47.106935Z","iopub.status.idle":"2022-10-14T16:20:47.115339Z","shell.execute_reply.started":"2022-10-14T16:20:47.106903Z","shell.execute_reply":"2022-10-14T16:20:47.114157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ppi_filter.to_csv('df_ppi_filter.csv')","metadata":{"execution":{"iopub.status.busy":"2022-10-14T16:17:03.243648Z","iopub.execute_input":"2022-10-14T16:17:03.244010Z","iopub.status.idle":"2022-10-14T16:17:03.745212Z","shell.execute_reply.started":"2022-10-14T16:17:03.243978Z","shell.execute_reply":"2022-10-14T16:17:03.744163Z"},"trusted":true},"execution_count":null,"outputs":[]}]}