{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\n!pip3 install scanpy[leiden] gensim","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:00:56.904815Z","iopub.execute_input":"2022-12-14T21:00:56.905322Z","iopub.status.idle":"2022-12-14T21:01:15.499166Z","shell.execute_reply.started":"2022-12-14T21:00:56.905212Z","shell.execute_reply":"2022-12-14T21:01:15.497258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport os\nimport sys\n\nimport anndata as ad\nimport numpy as np\nimport pandas as pd\nimport scanpy as sc\n\nfrom gensim.models import word2vec\nfrom tqdm.notebook import tqdm\n\nDATA_DIR = \"/kaggle/input/openproblemsmultimodalanndata\"","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:01:15.502073Z","iopub.execute_input":"2022-12-14T21:01:15.502487Z","iopub.status.idle":"2022-12-14T21:01:18.981227Z","shell.execute_reply.started":"2022-12-14T21:01:15.502445Z","shell.execute_reply":"2022-12-14T21:01:18.979601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Input data is cell-normalised and log1p transformed","metadata":{}},{"cell_type":"code","source":"# Cite\nFP_CITE_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_cite_inputs_values.sparse.ann.h5\")\nFP_CITE_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_cite_targets_values.sparse.ann.h5\")\nFP_CITE_TEST_INPUTS = os.path.join(DATA_DIR,\"test_cite_inputs_values.sparse.ann.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:01:18.982922Z","iopub.execute_input":"2022-12-14T21:01:18.983321Z","iopub.status.idle":"2022-12-14T21:01:18.989816Z","shell.execute_reply.started":"2022-12-14T21:01:18.983285Z","shell.execute_reply":"2022-12-14T21:01:18.988419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR_RAW = \"/kaggle/input/msci-raw-counts-anndata\"\n# Cite\nFP_CITE_RAW_TRAIN_INPUTS = os.path.join(DATA_DIR_RAW,\"train_cite_inputs_raw.ann.h5\")\nFP_CITE_RAW_TRAIN_TARGETS = os.path.join(DATA_DIR_RAW,\"train_cite_targets_raw.ann.h5\")\nFP_CITE_RAW_TEST_INPUTS = os.path.join(DATA_DIR_RAW,\"test_cite_inputs_raw.ann.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:01:18.992408Z","iopub.execute_input":"2022-12-14T21:01:18.992824Z","iopub.status.idle":"2022-12-14T21:01:19.026360Z","shell.execute_reply.started":"2022-12-14T21:01:18.992786Z","shell.execute_reply":"2022-12-14T21:01:19.024473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_fp = \"/kaggle/input/open-problems-multimodal/metadata.csv\"\nadd_metadata_fp = \"/kaggle/input/open-problems-multimodal/metadata_cite_day_2_donor_27678.csv\"\nmetadata = pd.read_csv(metadata_fp)\nadd_metadata = pd.read_csv(add_metadata_fp)\nmetadata = pd.concat([metadata, add_metadata])\nmetadata.set_index(\"cell_id\", inplace=True)\nmetadata.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:01:19.028034Z","iopub.execute_input":"2022-12-14T21:01:19.028438Z","iopub.status.idle":"2022-12-14T21:01:19.561011Z","shell.execute_reply.started":"2022-12-14T21:01:19.028390Z","shell.execute_reply":"2022-12-14T21:01:19.559750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def soft_max_by_row(x):\n    exp_x = np.exp(x)\n    return exp_x/np.sum(exp_x, axis=1, keepdims=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:05:20.022879Z","iopub.execute_input":"2022-12-14T21:05:20.023380Z","iopub.status.idle":"2022-12-14T21:05:20.030417Z","shell.execute_reply.started":"2022-12-14T21:05:20.023332Z","shell.execute_reply":"2022-12-14T21:05:20.028593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def annotate(adata: ad.AnnData, metadata: pd.DataFrame, f_donors: [str]):\n    adata.obs[\"cell_type\"] = metadata[\"cell_type\"][adata.obs_names]\n    adata.obs[\"day\"] = metadata[\"day\"][adata.obs_names].astype(str)\n    adata.obs[\"donor\"] = metadata[\"donor\"][adata.obs_names].astype(str)\n    adata.obs[\"gender\"] = \"m\"\n    adata.obs.loc[adata.obs[\"donor\"].isin(f_donors), \"gender\"] = \"f\"\n    adata.obs[\"n_genes_counts\"] = (adata.X > 0).sum(axis=1).A1\n    adata.obs[\"read_depth\"] = adata.X.sum(axis=1).A1","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:01:19.570672Z","iopub.execute_input":"2022-12-14T21:01:19.571118Z","iopub.status.idle":"2022-12-14T21:01:19.584255Z","shell.execute_reply.started":"2022-12-14T21:01:19.571051Z","shell.execute_reply":"2022-12-14T21:01:19.583130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_target = sc.read_h5ad(FP_CITE_TRAIN_TARGETS)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:01:19.585422Z","iopub.execute_input":"2022-12-14T21:01:19.586010Z","iopub.status.idle":"2022-12-14T21:01:20.717406Z","shell.execute_reply.started":"2022-12-14T21:01:19.585975Z","shell.execute_reply":"2022-12-14T21:01:20.716136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_top = 100\nemb_size = 128\nwindow_size = 50\nw2v_algorithm = 1  # 1 for skip-gram; otherwise CBOW\nnegative = 30  # negative sampling","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:01:20.719093Z","iopub.execute_input":"2022-12-14T21:01:20.719469Z","iopub.status.idle":"2022-12-14T21:01:20.725674Z","shell.execute_reply.started":"2022-12-14T21:01:20.719434Z","shell.execute_reply":"2022-12-14T21:01:20.724176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Top N gene coordinates\nYou can either take Top N higly expressed genes for each cell or get top HVG (with batch correction)","metadata":{}},{"cell_type":"markdown","source":"## Higly expressed HVG","metadata":{}},{"cell_type":"code","source":"cite_raw_train = sc.read_h5ad(FP_CITE_RAW_TRAIN_INPUTS)\ncite_raw_test = sc.read_h5ad(FP_CITE_RAW_TEST_INPUTS)\ncite_raw = ad.concat([cite_raw_train, cite_raw_test], merge=\"same\")\ndel cite_raw_train\ndel cite_raw_test\ngc.collect()\nannotate(cite_raw, metadata, f_donors = [\"13176\", ])\nadata = cite_raw\nadata","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:01:20.729287Z","iopub.execute_input":"2022-12-14T21:01:20.729716Z","iopub.status.idle":"2022-12-14T21:02:19.958615Z","shell.execute_reply.started":"2022-12-14T21:01:20.729678Z","shell.execute_reply":"2022-12-14T21:02:19.957184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.experimental.pp.highly_variable_genes(adata, theta=100/14, batch_key=\"donor\", n_top_genes=3000)\nsum(adata.var[\"highly_variable_intersection\"])","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:02:19.960436Z","iopub.execute_input":"2022-12-14T21:02:19.961805Z","iopub.status.idle":"2022-12-14T21:04:21.184017Z","shell.execute_reply.started":"2022-12-14T21:02:19.961746Z","shell.execute_reply":"2022-12-14T21:04:21.182254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.normalize_total(adata, target_sum=1e4)\nadata = adata[:, adata.var_names[adata.var[\"highly_variable_intersection\"]]]\n\nsc.pp.log1p(adata)\n# We will scale gene variance\n# use zero_center=False to leave adata sparse efficient\nsc.pp.scale(adata, max_value=10) #, zero_center=False)\n\nadata","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:04:21.187529Z","iopub.execute_input":"2022-12-14T21:04:21.188081Z","iopub.status.idle":"2022-12-14T21:04:32.627651Z","shell.execute_reply.started":"2022-12-14T21:04:21.188032Z","shell.execute_reply":"2022-12-14T21:04:32.626235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_fn = os.path.basename(FP_CITE_TRAIN_INPUTS).replace(\".ann.h5\", f\".top{N_top}.npy\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:04:32.629871Z","iopub.execute_input":"2022-12-14T21:04:32.630801Z","iopub.status.idle":"2022-12-14T21:04:32.637302Z","shell.execute_reply.started":"2022-12-14T21:04:32.630747Z","shell.execute_reply":"2022-12-14T21:04:32.636123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sort_array = np.argpartition(-adata.X, N_top, axis=-1)[:,0:N_top]\nnp.save(output_fn, sort_array, allow_pickle=True)\nsort_array.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:04:58.121604Z","iopub.execute_input":"2022-12-14T21:04:58.122115Z","iopub.status.idle":"2022-12-14T21:05:02.540880Z","shell.execute_reply.started":"2022-12-14T21:04:58.122077Z","shell.execute_reply":"2022-12-14T21:05:02.539496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cell tokens","metadata":{}},{"cell_type":"code","source":"top_genes_expression = adata.X[np.repeat(np.arange(0, sort_array.shape[0]), sort_array.shape[1]), sort_array.flatten()].reshape((-1, sort_array.shape[-1]))\ntop_genes_expression.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:05:02.543378Z","iopub.execute_input":"2022-12-14T21:05:02.544527Z","iopub.status.idle":"2022-12-14T21:05:02.940071Z","shell.execute_reply.started":"2022-12-14T21:05:02.544488Z","shell.execute_reply":"2022-12-14T21:05:02.938674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_gene_weights = soft_max_by_row(top_genes_expression)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:05:24.482161Z","iopub.execute_input":"2022-12-14T21:05:24.482655Z","iopub.status.idle":"2022-12-14T21:05:24.546117Z","shell.execute_reply.started":"2022-12-14T21:05:24.482601Z","shell.execute_reply":"2022-12-14T21:05:24.544901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_tokens = sort_array.astype(\"str\")\ncell_tokens[0]","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:05:28.591817Z","iopub.execute_input":"2022-12-14T21:05:28.592259Z","iopub.status.idle":"2022-12-14T21:05:35.525559Z","shell.execute_reply.started":"2022-12-14T21:05:28.592211Z","shell.execute_reply":"2022-12-14T21:05:35.523954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Word2Vec does not work with np array\ncell_tokens_list = []\n\nfor i in cell_tokens:\n    cell_tokens_list.append(list(i))\n\nlen(cell_tokens_list)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:05:35.527755Z","iopub.execute_input":"2022-12-14T21:05:35.528122Z","iopub.status.idle":"2022-12-14T21:05:42.993282Z","shell.execute_reply.started":"2022-12-14T21:05:35.528088Z","shell.execute_reply":"2022-12-14T21:05:42.991882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index_to_key_fn = f\"w2v_gene_embeddings_topNexpr_{N_top}_emb_s_{emb_size}_ws_{window_size}_{'skip_gram' if w2v_algorithm else 'CBOW'}_neg_{negative}_index_to_key\"\ngene_embeddings_fn = f\"w2v_gene_embeddings_topNexpr_{N_top}_emb_s_{emb_size}_ws_{window_size}_{'skip_gram' if w2v_algorithm else 'CBOW'}_neg_{negative}.npy\"","metadata":{"execution":{"iopub.status.busy":"2022-12-14T21:05:42.995168Z","iopub.execute_input":"2022-12-14T21:05:42.995674Z","iopub.status.idle":"2022-12-14T21:05:43.001351Z","shell.execute_reply.started":"2022-12-14T21:05:42.995621Z","shell.execute_reply":"2022-12-14T21:05:42.999992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run word2vec\nIt seems that Word2Vec does not use GPU :(","metadata":{}},{"cell_type":"code","source":"# model = word2vec.Word2Vec(\n#     sentences=cell_tokens_list, vector_size=emb_size, window=window_size, min_count=1,\n#     sg=w2v_algorithm, negative=negative\n# )\n# word_vectors = model.wv\n\n# word_vectors_npy = np.array([word_vectors[key] for key in word_vectors.index_to_key])    \n# index_to_key = word_vectors.index_to_key","metadata":{"_uuid":"051d70d956493feee0c6d64651c6a088724dca2a","_execution_state":"idle","execution":{"iopub.status.busy":"2022-12-14T21:05:53.483139Z","iopub.execute_input":"2022-12-14T21:05:53.483598Z","iopub.status.idle":"2022-12-14T23:14:19.213401Z","shell.execute_reply.started":"2022-12-14T21:05:53.483558Z","shell.execute_reply":"2022-12-14T23:14:19.210928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### presave embeddings for further usage","metadata":{}},{"cell_type":"code","source":"# with open(index_to_key_fn, \"w\") as f:\n#     f.write(\",\".join(word_vectors.index_to_key))\n    \n# np.save(gene_embeddings_fn, word_vectors_npy, allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:14:19.217544Z","iopub.execute_input":"2022-12-14T23:14:19.218020Z","iopub.status.idle":"2022-12-14T23:14:19.230147Z","shell.execute_reply.started":"2022-12-14T23:14:19.217980Z","shell.execute_reply":"2022-12-14T23:14:19.228716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### load presaved embeddings","metadata":{}},{"cell_type":"code","source":"index_to_key_fp = os.path.join(PRESAVED_DIR, index_to_key_fn)\ngene_embeddings_fp = os.path.join(PRESAVED_DIR, gene_embeddings_fn)\n\nwith open(index_to_key_fp) as f:\n    index_to_key = f.read().split(\",\")\n\nwith open(gene_embeddings_fp) as f:\n    word_vectors_npy = np.load(f)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Count cell embeddings as softmaxed gene expression weigthed sum of gene embeddings","metadata":{}},{"cell_type":"code","source":"w2v_emb = np.zeros((sort_array.shape[0], emb_size))\n\n\nfor i in tqdm(range(sort_array.shape[0])):\n    # [N_top,]\n    cell_gene_weigth = cell_gene_weights[i]  #.A1\n    # [emb_size, N_top]\n    cell_gene_w2v = np.stack([word_vectors[gene_token] for gene_token in cell_tokens[i]], axis=1)\n    # [emb_size,]\n    w2v_emb[i] = cell_gene_w2v @ cell_gene_weigth","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:17:01.287339Z","iopub.execute_input":"2022-12-14T23:17:01.287862Z","iopub.status.idle":"2022-12-14T23:19:29.127230Z","shell.execute_reply.started":"2022-12-14T23:17:01.287822Z","shell.execute_reply":"2022-12-14T23:19:29.125883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w2v_emb.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:21:57.263939Z","iopub.execute_input":"2022-12-14T23:21:57.264424Z","iopub.status.idle":"2022-12-14T23:21:57.274114Z","shell.execute_reply.started":"2022-12-14T23:21:57.264391Z","shell.execute_reply":"2022-12-14T23:21:57.272668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. PCA on w2v cell embeddings","metadata":{}},{"cell_type":"code","source":"sc.set_figure_params(dpi=150, facecolor=\"white\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:22:01.498964Z","iopub.execute_input":"2022-12-14T23:22:01.499434Z","iopub.status.idle":"2022-12-14T23:22:01.508706Z","shell.execute_reply.started":"2022-12-14T23:22:01.499393Z","shell.execute_reply":"2022-12-14T23:22:01.507149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adata.obsm[\"X_pca\"] = sc.tl.pca(w2v_emb, n_comps=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:22:03.588424Z","iopub.execute_input":"2022-12-14T23:22:03.588873Z","iopub.status.idle":"2022-12-14T23:22:07.739227Z","shell.execute_reply.started":"2022-12-14T23:22:03.588838Z","shell.execute_reply":"2022-12-14T23:22:07.738136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.pca(adata, color=\"cell_type\", dimensions=[(0, 1), (1, 2)], title=f\"PCA topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:22:09.871334Z","iopub.execute_input":"2022-12-14T23:22:09.871777Z","iopub.status.idle":"2022-12-14T23:22:12.755358Z","shell.execute_reply.started":"2022-12-14T23:22:09.871738Z","shell.execute_reply":"2022-12-14T23:22:12.753745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. UMAP on PCA on w2v embeddings","metadata":{}},{"cell_type":"code","source":"sc.pp.neighbors(adata, n_pcs=15, n_neighbors=10)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:34:57.035463Z","iopub.execute_input":"2022-12-14T23:34:57.035957Z","iopub.status.idle":"2022-12-14T23:35:11.840570Z","shell.execute_reply.started":"2022-12-14T23:34:57.035916Z","shell.execute_reply":"2022-12-14T23:35:11.839316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.tl.umap(adata, min_dist=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:35:11.843060Z","iopub.execute_input":"2022-12-14T23:35:11.844054Z","iopub.status.idle":"2022-12-14T23:36:49.809585Z","shell.execute_reply.started":"2022-12-14T23:35:11.844011Z","shell.execute_reply":"2022-12-14T23:36:49.808017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### by cell type","metadata":{}},{"cell_type":"code","source":"sc.pl.umap(adata, color=\"cell_type\", frameon=False, title=f\"UMAP topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:37:02.265213Z","iopub.execute_input":"2022-12-14T23:37:02.265668Z","iopub.status.idle":"2022-12-14T23:37:03.684529Z","shell.execute_reply.started":"2022-12-14T23:37:02.265615Z","shell.execute_reply":"2022-12-14T23:37:03.683333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### by donor","metadata":{}},{"cell_type":"code","source":"sc.pl.umap(adata, color=\"donor\", frameon=False, title=f\"UMAP topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:37:20.047445Z","iopub.execute_input":"2022-12-14T23:37:20.049090Z","iopub.status.idle":"2022-12-14T23:37:21.347266Z","shell.execute_reply.started":"2022-12-14T23:37:20.049039Z","shell.execute_reply":"2022-12-14T23:37:21.346162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### by day","metadata":{}},{"cell_type":"code","source":"sc.pl.umap(adata, color=\"day\", frameon=False, title=f\"UMAP topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:37:29.672796Z","iopub.execute_input":"2022-12-14T23:37:29.673210Z","iopub.status.idle":"2022-12-14T23:37:30.981073Z","shell.execute_reply.started":"2022-12-14T23:37:29.673177Z","shell.execute_reply":"2022-12-14T23:37:30.978914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### by gender","metadata":{}},{"cell_type":"code","source":"sc.pl.umap(adata, color=\"gender\", frameon=False, title=f\"UMAP topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:37:40.141615Z","iopub.execute_input":"2022-12-14T23:37:40.142138Z","iopub.status.idle":"2022-12-14T23:37:41.487755Z","shell.execute_reply.started":"2022-12-14T23:37:40.142082Z","shell.execute_reply":"2022-12-14T23:37:41.486047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### by non-zero gene count","metadata":{}},{"cell_type":"code","source":"sc.pl.umap(adata, color=\"n_genes_counts\", frameon=False, title=f\"UMAP topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:37:57.866522Z","iopub.execute_input":"2022-12-14T23:37:57.866962Z","iopub.status.idle":"2022-12-14T23:37:58.883117Z","shell.execute_reply.started":"2022-12-14T23:37:57.866929Z","shell.execute_reply":"2022-12-14T23:37:58.881341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"G1S_genes_Tirosh = ['ENSG00000100297', 'ENSG00000132646', 'ENSG00000176890', 'ENSG00000168496', 'ENSG00000073111', 'ENSG00000104738', 'ENSG00000167325', 'ENSG00000076248', 'ENSG00000131153', 'ENSG00000076003', 'ENSG00000144354', 'ENSG00000143476', 'ENSG00000198056', 'ENSG00000276043', 'ENSG00000151725', 'ENSG00000119969', 'ENSG00000049541', 'ENSG00000117748', 'ENSG00000132780', 'ENSG00000111247', 'ENSG00000112312', 'ENSG00000092470', 'ENSG00000163950', 'ENSG00000175305', 'ENSG00000012963', 'ENSG00000077514', 'ENSG00000095002', 'ENSG00000156802', 'ENSG00000051180', 'ENSG00000171848', 'ENSG00000093009', 'ENSG00000094804', 'ENSG00000174371', 'ENSG00000075131', 'ENSG00000136982', 'ENSG00000197299', 'ENSG00000118412', 'ENSG00000162607', 'ENSG00000092853', 'ENSG00000101868', 'ENSG00000159259', 'ENSG00000136492', 'ENSG00000129173']\nG2M_genes_Tirosh = ['ENSG00000164104', 'ENSG00000170312', 'ENSG00000137804', 'ENSG00000175063', 'ENSG00000089685', 'ENSG00000088325', 'ENSG00000131747', 'ENSG00000080986', 'ENSG00000123975', 'ENSG00000143228', 'ENSG00000173207', 'ENSG00000148773', 'ENSG00000120802', 'ENSG00000117724', 'ENSG00000013810', 'ENSG00000129195', 'ENSG00000113810', 'ENSG00000157456', 'ENSG00000169607', 'ENSG00000136108', 'ENSG00000178999', 'ENSG00000169679', 'ENSG00000138160', 'ENSG00000143401', 'ENSG00000188229', 'ENSG00000075218', 'ENSG00000138182', 'ENSG00000123485', 'ENSG00000111665', 'ENSG00000189159', 'ENSG00000117399', 'ENSG00000112742', 'ENSG00000158402', 'ENSG00000142945', 'ENSG00000100401', 'ENSG00000010292', 'ENSG00000126787', 'ENSG00000184661', 'ENSG00000134690', 'ENSG00000114346', 'ENSG00000137807', 'ENSG00000072571', 'ENSG00000087586', 'ENSG00000134222', 'ENSG00000011426', 'ENSG00000143815', 'ENSG00000175216', 'ENSG00000138778', 'ENSG00000102974', 'ENSG00000117650', 'ENSG00000092140', 'ENSG00000139354', 'ENSG00000094916', 'ENSG00000115163']","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:38:11.871289Z","iopub.execute_input":"2022-12-14T23:38:11.871762Z","iopub.status.idle":"2022-12-14T23:38:11.882825Z","shell.execute_reply.started":"2022-12-14T23:38:11.871725Z","shell.execute_reply":"2022-12-14T23:38:11.881484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adata.var[\"ensembl_ids\"] = [t.split('_')[0] for t in adata.var_names]\nadata.var[\"gene_names\"] = [t.split('_')[1] for t in adata.var_names]","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:38:13.500218Z","iopub.execute_input":"2022-12-14T23:38:13.500737Z","iopub.status.idle":"2022-12-14T23:38:13.513870Z","shell.execute_reply.started":"2022-12-14T23:38:13.500681Z","shell.execute_reply":"2022-12-14T23:38:13.512791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adata.obs[\"G1S_genes_Tirosh_mean\"] = adata[:,adata.var[\"ensembl_ids\"].isin(G1S_genes_Tirosh)].X.sum(axis=1) / len(G1S_genes_Tirosh)\nadata.obs[\"G2M_genes_Tirosh_mean\"] = adata[:,adata.var[\"ensembl_ids\"].isin(G2M_genes_Tirosh)].X.sum(axis=1) / len(G2M_genes_Tirosh)\nadata.obs[\"G1S+2M_genes_Tirosh_mean\"] = adata[:,adata.var[\"ensembl_ids\"].isin(G1S_genes_Tirosh + G2M_genes_Tirosh)].X.sum(axis=1) / len(G1S_genes_Tirosh + G2M_genes_Tirosh)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:38:38.932878Z","iopub.execute_input":"2022-12-14T23:38:38.933368Z","iopub.status.idle":"2022-12-14T23:38:39.636926Z","shell.execute_reply.started":"2022-12-14T23:38:38.933325Z","shell.execute_reply":"2022-12-14T23:38:39.635518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### by mean expression of proliferation genes","metadata":{}},{"cell_type":"code","source":"sc.pl.umap(adata, color=\"G1S_genes_Tirosh_mean\", frameon=False, title=f\"UMAP G1S_genes_Tirosh_mean topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:38:41.497188Z","iopub.execute_input":"2022-12-14T23:38:41.498186Z","iopub.status.idle":"2022-12-14T23:38:42.554447Z","shell.execute_reply.started":"2022-12-14T23:38:41.498124Z","shell.execute_reply":"2022-12-14T23:38:42.553201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.umap(adata, color=\"G2M_genes_Tirosh_mean\", frameon=False, title=f\"UMAP G2M_genes_Tirosh_mean topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:38:50.337034Z","iopub.execute_input":"2022-12-14T23:38:50.337453Z","iopub.status.idle":"2022-12-14T23:38:51.368912Z","shell.execute_reply.started":"2022-12-14T23:38:50.337420Z","shell.execute_reply":"2022-12-14T23:38:51.367404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### by CD36 gene","metadata":{}},{"cell_type":"code","source":"sc.pl.umap(adata, color=[\"ENSG00000135218_CD36\", \"cell_type\"], frameon=False, title=f\"UMAP ENSG00000135218 topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:39:22.905860Z","iopub.execute_input":"2022-12-14T23:39:22.906313Z","iopub.status.idle":"2022-12-14T23:39:25.178564Z","shell.execute_reply.started":"2022-12-14T23:39:22.906278Z","shell.execute_reply":"2022-12-14T23:39:25.176285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### by proteins","metadata":{}},{"cell_type":"code","source":"cite_w_target = adata[cite_target.obs_names].copy()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:39:43.589007Z","iopub.execute_input":"2022-12-14T23:39:43.589459Z","iopub.status.idle":"2022-12-14T23:39:44.247075Z","shell.execute_reply.started":"2022-12-14T23:39:43.589413Z","shell.execute_reply":"2022-12-14T23:39:44.245717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prots_df = pd.DataFrame(cite_target.X.A, columns=cite_target.var_names, index=cite_w_target.obs.index)\ncite_w_target.obs = pd.concat([cite_w_target.obs, prots_df], axis=1)\n\ncite_w_target.obs.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:39:49.293345Z","iopub.execute_input":"2022-12-14T23:39:49.293822Z","iopub.status.idle":"2022-12-14T23:39:49.646122Z","shell.execute_reply.started":"2022-12-14T23:39:49.293776Z","shell.execute_reply":"2022-12-14T23:39:49.644838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prot_names = list(cite_target.var_names)\nprot_names.sort()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:39:53.468907Z","iopub.execute_input":"2022-12-14T23:39:53.469335Z","iopub.status.idle":"2022-12-14T23:39:53.475338Z","shell.execute_reply.started":"2022-12-14T23:39:53.469294Z","shell.execute_reply":"2022-12-14T23:39:53.474172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.set_figure_params(dpi=40)\nsc.pl.umap(cite_w_target, color=[\"cell_type\"] + prot_names, frameon=False, ncols=12)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:39:57.682072Z","iopub.execute_input":"2022-12-14T23:39:57.682496Z","iopub.status.idle":"2022-12-14T23:40:55.276398Z","shell.execute_reply.started":"2022-12-14T23:39:57.682465Z","shell.execute_reply":"2022-12-14T23:40:55.275007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.set_figure_params(dpi=150)\nsc.pl.umap(cite_w_target, color=[\"CD36\", \"ENSG00000135218_CD36\", \"cell_type\"], frameon=False, title=f\"UMAP CD36 topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:40:55.278601Z","iopub.execute_input":"2022-12-14T23:40:55.278999Z","iopub.status.idle":"2022-12-14T23:40:58.978823Z","shell.execute_reply.started":"2022-12-14T23:40:55.278963Z","shell.execute_reply":"2022-12-14T23:40:58.976375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Investigation of w2v gene embeddings\nLet's create anndata object with gene embeddings","metadata":{}},{"cell_type":"code","source":"genes = ad.AnnData(word_vectors_npy)\ngenes.obs_names = [adata.var_names[int(key)] for key in index_to_key]  # gene names\ngenes","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:41:45.647913Z","iopub.execute_input":"2022-12-14T23:41:45.648426Z","iopub.status.idle":"2022-12-14T23:41:45.666872Z","shell.execute_reply.started":"2022-12-14T23:41:45.648387Z","shell.execute_reply":"2022-12-14T23:41:45.665571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.tl.pca(genes, n_comps=50)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:41:59.114428Z","iopub.execute_input":"2022-12-14T23:41:59.114915Z","iopub.status.idle":"2022-12-14T23:41:59.192364Z","shell.execute_reply.started":"2022-12-14T23:41:59.114868Z","shell.execute_reply":"2022-12-14T23:41:59.190700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.set_figure_params(dpi=150, facecolor=\"white\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:42:06.143579Z","iopub.execute_input":"2022-12-14T23:42:06.144037Z","iopub.status.idle":"2022-12-14T23:42:06.150079Z","shell.execute_reply.started":"2022-12-14T23:42:06.143998Z","shell.execute_reply":"2022-12-14T23:42:06.148872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.pca(genes, dimensions=[(0, 1), (0, 2)])  #, title=f\"PCA topN={N_top} w2v emb_s={emb_size} ws={window_size} {'skip-gram' if w2v_algorithm else 'CBOW'} neg={negative}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:42:15.736515Z","iopub.execute_input":"2022-12-14T23:42:15.737033Z","iopub.status.idle":"2022-12-14T23:42:16.238398Z","shell.execute_reply.started":"2022-12-14T23:42:15.736992Z","shell.execute_reply":"2022-12-14T23:42:16.236972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.pca_variance_ratio(genes)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:42:25.185166Z","iopub.execute_input":"2022-12-14T23:42:25.185643Z","iopub.status.idle":"2022-12-14T23:42:25.601459Z","shell.execute_reply.started":"2022-12-14T23:42:25.185586Z","shell.execute_reply":"2022-12-14T23:42:25.600153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.neighbors(genes, n_pcs=15, n_neighbors=10)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:42:31.706515Z","iopub.execute_input":"2022-12-14T23:42:31.706982Z","iopub.status.idle":"2022-12-14T23:42:33.404129Z","shell.execute_reply.started":"2022-12-14T23:42:31.706945Z","shell.execute_reply":"2022-12-14T23:42:33.403111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.tl.umap(genes, min_dist=1)\nsc.pl.umap(genes, frameon=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:42:35.443659Z","iopub.execute_input":"2022-12-14T23:42:35.445195Z","iopub.status.idle":"2022-12-14T23:42:39.648402Z","shell.execute_reply.started":"2022-12-14T23:42:35.445145Z","shell.execute_reply":"2022-12-14T23:42:39.647106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Will colour umap by cell type dif expressed genes ","metadata":{}},{"cell_type":"code","source":"DIF_EXPRESSED_DIR = \"/kaggle/input/cite-differential-expression-genes\"\nos.listdir(DIF_EXPRESSED_DIR)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:42:46.075442Z","iopub.execute_input":"2022-12-14T23:42:46.075861Z","iopub.status.idle":"2022-12-14T23:42:46.090020Z","shell.execute_reply.started":"2022-12-14T23:42:46.075827Z","shell.execute_reply":"2022-12-14T23:42:46.088826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_types = adata.obs[\"cell_type\"].unique()\ncell_types","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:42:58.246319Z","iopub.execute_input":"2022-12-14T23:42:58.246799Z","iopub.status.idle":"2022-12-14T23:42:58.256975Z","shell.execute_reply.started":"2022-12-14T23:42:58.246761Z","shell.execute_reply":"2022-12-14T23:42:58.256078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for cell_type in cell_types:\n    de_genes_fp = os.path.join(DIF_EXPRESSED_DIR, f\"wilcoxon_{cell_type}_vs_rest.csv\")\n    de_genes = pd.read_csv(de_genes_fp, index_col=0)\n    de_genes.set_index(\"names\", inplace=True)\n#     interesting_de_genes = de_genes[(de_genes.pvals_adj < 0.01) & de_genes.names.isin(genes.obs_names)]\n#     interesting_de_genes.set_index(\"names\")\n    genes.obs[f\"de_logfoldchanges_{cell_type}\"] = de_genes.loc[genes.obs_names, \"logfoldchanges\"]\n    genes.obs[f\"de_pvals_adj_{cell_type}\"] = de_genes.loc[genes.obs_names, \"pvals_adj\"]\n\ngenes.obs.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:43:07.073744Z","iopub.execute_input":"2022-12-14T23:43:07.074188Z","iopub.status.idle":"2022-12-14T23:43:07.594283Z","shell.execute_reply.started":"2022-12-14T23:43:07.074152Z","shell.execute_reply":"2022-12-14T23:43:07.592990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_types_lfc = [f\"de_logfoldchanges_{cell_type}\" for cell_type in cell_types]\ncell_types_pva = [f\"de_pvals_adj_{cell_type}\" for cell_type in cell_types]","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:43:11.441577Z","iopub.execute_input":"2022-12-14T23:43:11.442042Z","iopub.status.idle":"2022-12-14T23:43:11.449157Z","shell.execute_reply.started":"2022-12-14T23:43:11.442004Z","shell.execute_reply":"2022-12-14T23:43:11.447761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.umap(genes, frameon=False, color=cell_types_lfc, vmin=-10, vmax=10)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:43:14.107156Z","iopub.execute_input":"2022-12-14T23:43:14.107606Z","iopub.status.idle":"2022-12-14T23:43:16.298282Z","shell.execute_reply.started":"2022-12-14T23:43:14.107569Z","shell.execute_reply":"2022-12-14T23:43:16.297206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for cell_type in cell_types:\n    de_genes_fp = os.path.join(DIF_EXPRESSED_DIR, f\"wilcoxon_{cell_type}_vs_rest.csv\")\n    de_genes = pd.read_csv(de_genes_fp, index_col=0)\n    interesting_de_genes = de_genes[(de_genes.pvals_adj < 0.01) & (de_genes.logfoldchanges > 1) & de_genes.names.isin(genes.obs_names)]\n    interesting_de_genes.set_index(\"names\", inplace=True)\n    genes.obs[f\"de_significant_{cell_type}\"] = 0\n    genes.obs.loc[genes.obs_names.isin(interesting_de_genes.index), f\"de_significant_{cell_type}\"] = 1\n\ngenes.obs.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:45:16.549790Z","iopub.execute_input":"2022-12-14T23:45:16.550288Z","iopub.status.idle":"2022-12-14T23:45:16.870363Z","shell.execute_reply.started":"2022-12-14T23:45:16.550254Z","shell.execute_reply":"2022-12-14T23:45:16.869407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_types_significant = [f\"de_significant_{cell_type}\" for cell_type in cell_types]","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:45:26.162571Z","iopub.execute_input":"2022-12-14T23:45:26.163088Z","iopub.status.idle":"2022-12-14T23:45:26.169644Z","shell.execute_reply.started":"2022-12-14T23:45:26.163050Z","shell.execute_reply":"2022-12-14T23:45:26.168124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.umap(genes, frameon=False, color=cell_types_significant)","metadata":{"execution":{"iopub.status.busy":"2022-12-14T23:45:29.467059Z","iopub.execute_input":"2022-12-14T23:45:29.467500Z","iopub.status.idle":"2022-12-14T23:45:31.574470Z","shell.execute_reply.started":"2022-12-14T23:45:29.467466Z","shell.execute_reply":"2022-12-14T23:45:31.573064Z"},"trusted":true},"execution_count":null,"outputs":[]}]}