{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip3 install scanpy[leiden] opentsne scikit-misc","metadata":{"id":"BXFep4E1kXUx","outputId":"ceb49acc-f786-45f1-afcb-ca9a7139cbc8","execution":{"iopub.status.idle":"2022-12-06T18:09:38.631926Z","shell.execute_reply.started":"2022-12-06T18:09:19.103978Z","shell.execute_reply":"2022-12-06T18:09:38.630513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nDATA_DIR = \"/kaggle/input/openproblemsmultimodalanndata\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","id":"ydBe6telkXU8","execution":{"iopub.status.busy":"2022-12-06T18:09:38.634740Z","iopub.execute_input":"2022-12-06T18:09:38.635243Z","iopub.status.idle":"2022-12-06T18:09:38.642237Z","shell.execute_reply.started":"2022-12-06T18:09:38.635192Z","shell.execute_reply":"2022-12-06T18:09:38.640622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cite\nFP_CITE_TRAIN_INPUTS_ann = os.path.join(DATA_DIR,\"train_cite_inputs_values.sparse.ann.h5\")\nFP_CITE_TRAIN_TARGETS_ann = os.path.join(DATA_DIR,\"train_cite_targets_values.sparse.ann.h5\")\nFP_CITE_TEST_INPUTS_ann = os.path.join(DATA_DIR,\"test_cite_inputs_values.sparse.ann.h5\")","metadata":{"id":"WnT4wnCwkXVE","execution":{"iopub.status.busy":"2022-12-06T18:09:38.643958Z","iopub.execute_input":"2022-12-06T18:09:38.644430Z","iopub.status.idle":"2022-12-06T18:09:38.656409Z","shell.execute_reply.started":"2022-12-06T18:09:38.644396Z","shell.execute_reply":"2022-12-06T18:09:38.654989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR_RAW = \"/kaggle/input/msci-raw-counts-anndata\"\n# Cite\nFP_CITE_RAW_TRAIN_INPUTS = os.path.join(DATA_DIR_RAW,\"train_cite_inputs_raw.ann.h5\")\nFP_CITE_RAW_TRAIN_TARGETS = os.path.join(DATA_DIR_RAW,\"train_cite_targets_raw.ann.h5\")\nFP_CITE_RAW_TEST_INPUTS = os.path.join(DATA_DIR_RAW,\"test_cite_inputs_raw.ann.h5\")","metadata":{"id":"qIiAE2DGkXVO","execution":{"iopub.status.busy":"2022-12-06T18:09:38.659354Z","iopub.execute_input":"2022-12-06T18:09:38.659989Z","iopub.status.idle":"2022-12-06T18:09:38.673516Z","shell.execute_reply.started":"2022-12-06T18:09:38.659950Z","shell.execute_reply":"2022-12-06T18:09:38.672475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\nimport anndata as ad\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport scipy.stats as stats\nimport seaborn as sns\nimport scanpy as sc\n\nfrom openTSNE import TSNE\n\nsc.settings.verbosity = 3\nsc.set_figure_params(dpi=150)\nsns.set_style(\"ticks\")","metadata":{"id":"jyPoXS6rkXVQ","execution":{"iopub.status.busy":"2022-12-06T18:09:38.675251Z","iopub.execute_input":"2022-12-06T18:09:38.675748Z","iopub.status.idle":"2022-12-06T18:09:41.552195Z","shell.execute_reply.started":"2022-12-06T18:09:38.675710Z","shell.execute_reply":"2022-12-06T18:09:41.550917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metadata_fp = \"/kaggle/input/open-problems-multimodal/metadata.csv\"\nadd_metadata_fp = \"/kaggle/input/open-problems-multimodal/metadata_cite_day_2_donor_27678.csv\"\nmetadata = pd.read_csv(metadata_fp)\nadd_metadata = pd.read_csv(add_metadata_fp)\nmetadata = pd.concat([metadata, add_metadata])\nmetadata.set_index(\"cell_id\", inplace=True)\nmetadata.head()","metadata":{"id":"Ykg8qbKkkXVS","outputId":"46ccfa1e-efa8-4b6b-9720-952cecde0cd1","execution":{"iopub.status.busy":"2022-12-06T18:09:41.555778Z","iopub.execute_input":"2022-12-06T18:09:41.556209Z","iopub.status.idle":"2022-12-06T18:09:42.111513Z","shell.execute_reply.started":"2022-12-06T18:09:41.556173Z","shell.execute_reply":"2022-12-06T18:09:42.110698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def annotate(adata: ad.AnnData, metadata: pd.DataFrame, f_donors: [str]):\n    adata.obs[\"cell_type\"] = metadata[\"cell_type\"][adata.obs_names]\n    adata.obs[\"day\"] = metadata[\"day\"][adata.obs_names].astype(str)\n    adata.obs[\"donor\"] = metadata[\"donor\"][adata.obs_names].astype(str)\n    adata.obs[\"gender\"] = \"m\"\n    adata.obs.loc[adata.obs[\"donor\"].isin(f_donors), \"gender\"] = \"f\"\n    adata.obs[\"n_genes_counts\"] = (adata.X > 0).sum(axis=1).A1\n    adata.obs[\"read_depth\"] = adata.X.sum(axis=1).A1","metadata":{"id":"zzjh4x-3kXV_","execution":{"iopub.status.busy":"2022-12-06T18:09:44.307798Z","iopub.execute_input":"2022-12-06T18:09:44.308336Z","iopub.status.idle":"2022-12-06T18:09:44.318783Z","shell.execute_reply.started":"2022-12-06T18:09:44.308288Z","shell.execute_reply":"2022-12-06T18:09:44.317643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_top = 3000","metadata":{"id":"O1WX9_GikXWC","execution":{"iopub.status.busy":"2022-12-06T18:09:46.302190Z","iopub.execute_input":"2022-12-06T18:09:46.302653Z","iopub.status.idle":"2022-12-06T18:09:46.308372Z","shell.execute_reply.started":"2022-12-06T18:09:46.302607Z","shell.execute_reply":"2022-12-06T18:09:46.307013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_hvgs(fp: str, hvgs: list):\n    with open(fp, \"w\") as f:\n        f.write(\"\\n\".join(hvgs))","metadata":{"id":"0gNfSUlUkXWC","execution":{"iopub.status.busy":"2022-12-06T18:09:48.445297Z","iopub.execute_input":"2022-12-06T18:09:48.445750Z","iopub.status.idle":"2022-12-06T18:09:48.453305Z","shell.execute_reply.started":"2022-12-06T18:09:48.445710Z","shell.execute_reply":"2022-12-06T18:09:48.451586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_hvgs(fp: str):\n    with open(fp, \"r\") as f:\n        return f.read.splitlines()","metadata":{"id":"WRObCp2ykXWD","execution":{"iopub.status.busy":"2022-12-06T10:48:45.429434Z","iopub.execute_input":"2022-12-06T10:48:45.429886Z","iopub.status.idle":"2022-12-06T10:48:45.435664Z","shell.execute_reply.started":"2022-12-06T10:48:45.429838Z","shell.execute_reply":"2022-12-06T10:48:45.434199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# HVGs","metadata":{"id":"2fUtC2ZHkXWE"}},{"cell_type":"markdown","source":"## Seurat","metadata":{"id":"z_LwIdfgkXWG"}},{"cell_type":"code","source":"cite_train = sc.read_h5ad(FP_CITE_TRAIN_INPUTS_ann)\ncite_test = sc.read_h5ad(FP_CITE_TEST_INPUTS_ann)\ncite = ad.concat([cite_train, cite_test], merge=\"same\")\ndel cite_train\ndel cite_test\ngc.collect()\nannotate(cite, metadata, f_donors = [\"13176\", ])\ncite","metadata":{"execution":{"iopub.status.busy":"2022-12-06T09:55:36.163198Z","iopub.execute_input":"2022-12-06T09:55:36.163729Z","iopub.status.idle":"2022-12-06T09:56:42.991408Z","shell.execute_reply.started":"2022-12-06T09:55:36.163686Z","shell.execute_reply":"2022-12-06T09:56:42.990227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.highly_variable_genes(cite, flavor=\"seurat\", n_top_genes=N_top, batch_key=\"donor\")\nsc.pl.highly_variable_genes(cite)","metadata":{"id":"OiVKuMTQkXWH","outputId":"99560c22-1c95-487d-bf28-db48b1a284be","execution":{"iopub.status.busy":"2022-12-06T09:56:42.993300Z","iopub.execute_input":"2022-12-06T09:56:42.993660Z","iopub.status.idle":"2022-12-06T09:57:18.824838Z","shell.execute_reply.started":"2022-12-06T09:56:42.993627Z","shell.execute_reply":"2022-12-06T09:57:18.823618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hvg_seurat = cite.var[cite.var.highly_variable_intersection].index\nsave_hvgs(f\"HVG_cite_train_test_seurat_donor_{N_top}.txt\", hvg_seurat)\nlen(hvg_seurat)","metadata":{"id":"A7YfvPxdkXWI","execution":{"iopub.status.busy":"2022-12-06T09:57:18.826157Z","iopub.execute_input":"2022-12-06T09:57:18.826537Z","iopub.status.idle":"2022-12-06T09:57:18.836851Z","shell.execute_reply.started":"2022-12-06T09:57:18.826504Z","shell.execute_reply":"2022-12-06T09:57:18.835724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.scale(cite, zero_center=False, max_value=10)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T09:57:18.839788Z","iopub.execute_input":"2022-12-06T09:57:18.840232Z","iopub.status.idle":"2022-12-06T09:58:40.784525Z","shell.execute_reply.started":"2022-12-06T09:57:18.840193Z","shell.execute_reply":"2022-12-06T09:58:40.783382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.tl.pca(cite, n_comps=50)\nsc.pl.pca(cite, color=[\"cell_type\", \"donor\", \"day\"])","metadata":{"id":"uYFxu8c_kXWO","outputId":"2dffe617-95c7-4f86-8e6a-aa3b50e084c5","execution":{"iopub.status.busy":"2022-12-06T09:58:40.786204Z","iopub.execute_input":"2022-12-06T09:58:40.787303Z","iopub.status.idle":"2022-12-06T09:59:52.742493Z","shell.execute_reply.started":"2022-12-06T09:58:40.787261Z","shell.execute_reply":"2022-12-06T09:59:52.740788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.pca_variance_ratio(cite, n_pcs=50)","metadata":{"id":"TFfwEd6ukXWP","outputId":"42c7f628-8da8-4ee2-bb91-47ec2c4079b4","execution":{"iopub.status.busy":"2022-12-06T09:59:52.744503Z","iopub.execute_input":"2022-12-06T09:59:52.745246Z","iopub.status.idle":"2022-12-06T09:59:53.174389Z","shell.execute_reply.started":"2022-12-06T09:59:52.745207Z","shell.execute_reply":"2022-12-06T09:59:53.173215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.neighbors(cite, n_pcs=25, n_neighbors=30)","metadata":{"id":"iw2xv_nvkXWP","outputId":"d244c6a5-34ac-4cbe-ec9e-7d68e36c12d4","execution":{"iopub.status.busy":"2022-12-06T09:59:53.175823Z","iopub.execute_input":"2022-12-06T09:59:53.176517Z","iopub.status.idle":"2022-12-06T10:01:19.352561Z","shell.execute_reply.started":"2022-12-06T09:59:53.176482Z","shell.execute_reply":"2022-12-06T10:01:19.351453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.tl.umap(cite, min_dist=0.2)","metadata":{"id":"qCFHRpVskXWQ","outputId":"ab37769b-2a5e-41b4-8b73-99e769976c5b","execution":{"iopub.status.busy":"2022-12-06T10:01:19.354080Z","iopub.execute_input":"2022-12-06T10:01:19.356361Z","iopub.status.idle":"2022-12-06T10:04:08.885812Z","shell.execute_reply.started":"2022-12-06T10:01:19.356281Z","shell.execute_reply":"2022-12-06T10:04:08.884424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.umap(cite, color=[\"cell_type\", \"donor\", \"day\"], frameon=False)","metadata":{"id":"-DD-e4bfkXWQ","outputId":"9e914659-7f8c-4f68-87a1-620f642d8f7d","execution":{"iopub.status.busy":"2022-12-06T10:04:08.887727Z","iopub.execute_input":"2022-12-06T10:04:08.888293Z","iopub.status.idle":"2022-12-06T10:04:12.362468Z","shell.execute_reply.started":"2022-12-06T10:04:08.888244Z","shell.execute_reply":"2022-12-06T10:04:12.360812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cell ranger","metadata":{"id":"HRJlJTAukXWJ"}},{"cell_type":"code","source":"cite_train = sc.read_h5ad(FP_CITE_TRAIN_INPUTS_ann)\ncite_test = sc.read_h5ad(FP_CITE_TEST_INPUTS_ann)\ncite = ad.concat([cite_train, cite_test], merge=\"same\")\ndel cite_train\ndel cite_test\ngc.collect()\nannotate(cite, metadata, f_donors = [\"13176\", ])\ncite","metadata":{"execution":{"iopub.status.busy":"2022-12-06T10:04:12.369540Z","iopub.execute_input":"2022-12-06T10:04:12.370071Z","iopub.status.idle":"2022-12-06T10:05:25.030934Z","shell.execute_reply.started":"2022-12-06T10:04:12.370028Z","shell.execute_reply":"2022-12-06T10:05:25.029403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.highly_variable_genes(cite, flavor=\"cell_ranger\", n_top_genes=N_top, batch_key=\"donor\")\nsc.pl.highly_variable_genes(cite)","metadata":{"id":"NZlpC50DkXWJ","outputId":"0433f695-1bae-4cbe-aa73-351754dcfe31","execution":{"iopub.status.busy":"2022-12-06T10:05:25.032831Z","iopub.execute_input":"2022-12-06T10:05:25.033211Z","iopub.status.idle":"2022-12-06T10:05:46.819023Z","shell.execute_reply.started":"2022-12-06T10:05:25.033176Z","shell.execute_reply":"2022-12-06T10:05:46.817529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hvg_cr = cite.var[cite.var.highly_variable_intersection].index\nsave_hvgs(f\"HVG_cite_train_test_cell_ranger_donor_{N_top}.txt\", hvg_cr)\nlen(hvg_cr)","metadata":{"id":"stf3UEaYkXWK","execution":{"iopub.status.busy":"2022-12-06T10:05:46.820813Z","iopub.execute_input":"2022-12-06T10:05:46.821245Z","iopub.status.idle":"2022-12-06T10:05:46.833415Z","shell.execute_reply.started":"2022-12-06T10:05:46.821208Z","shell.execute_reply":"2022-12-06T10:05:46.832142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.tl.pca(cite, n_comps=50)\nsc.pl.pca(cite, color=[\"cell_type\", \"donor\", \"day\"])","metadata":{"id":"fEaE-1qAkXWS","outputId":"774d8389-3cf0-41b2-f91b-41a848e17ef5","execution":{"iopub.status.busy":"2022-12-06T10:05:46.835013Z","iopub.execute_input":"2022-12-06T10:05:46.835425Z","iopub.status.idle":"2022-12-06T10:06:50.458203Z","shell.execute_reply.started":"2022-12-06T10:05:46.835389Z","shell.execute_reply":"2022-12-06T10:06:50.456579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.neighbors(cite, n_pcs=25, n_neighbors=20)\nsc.tl.umap(cite, min_dist=0.2)\nsc.pl.umap(cite, color=[\"cell_type\", \"donor\", \"day\"], frameon=False)","metadata":{"id":"za2emSa_kXWS","outputId":"46035158-160f-48e0-f93b-4f144d59beb7","execution":{"iopub.status.busy":"2022-12-06T10:06:50.460215Z","iopub.execute_input":"2022-12-06T10:06:50.461173Z","iopub.status.idle":"2022-12-06T10:09:42.753259Z","shell.execute_reply.started":"2022-12-06T10:06:50.461114Z","shell.execute_reply":"2022-12-06T10:09:42.749247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Seurat_v3","metadata":{"id":"ZV1CRnFmkXWL"}},{"cell_type":"code","source":"cite_raw_train = sc.read_h5ad(FP_CITE_RAW_TRAIN_INPUTS)\ncite_raw_test = sc.read_h5ad(FP_CITE_RAW_TEST_INPUTS)\ncite_raw = ad.concat([cite_raw_train, cite_raw_test], merge=\"same\")\ndel cite_raw_train\ndel cite_raw_test\ngc.collect()\nannotate(cite_raw, metadata, f_donors = [\"13176\", ])\ncite_raw","metadata":{"execution":{"iopub.status.busy":"2022-12-06T10:09:42.755250Z","iopub.execute_input":"2022-12-06T10:09:42.755656Z","iopub.status.idle":"2022-12-06T10:11:11.444104Z","shell.execute_reply.started":"2022-12-06T10:09:42.755620Z","shell.execute_reply":"2022-12-06T10:11:11.442902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.highly_variable_genes(cite_raw, flavor=\"seurat_v3\", n_top_genes=N_top, batch_key=\"donor\")\nsc.pl.highly_variable_genes(cite_raw)","metadata":{"id":"jtrtDPMrkXWL","outputId":"091ae7c9-5813-45bc-eae0-6c56a30f77db","execution":{"iopub.status.busy":"2022-12-06T10:11:11.445528Z","iopub.execute_input":"2022-12-06T10:11:11.446007Z","iopub.status.idle":"2022-12-06T10:11:56.797263Z","shell.execute_reply.started":"2022-12-06T10:11:11.445938Z","shell.execute_reply":"2022-12-06T10:11:56.795892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hvg_seurat_v3 = cite_raw.var[cite_raw.var.highly_variable].index\nsave_hvgs(f\"HVG_cite_train_test_seurat_v3_donor_{N_top}.txt\", hvg_seurat_v3)\nlen(hvg_seurat_v3)","metadata":{"id":"aumfnpGGkXWM","execution":{"iopub.status.busy":"2022-12-06T10:11:56.799049Z","iopub.execute_input":"2022-12-06T10:11:56.799554Z","iopub.status.idle":"2022-12-06T10:11:56.812476Z","shell.execute_reply.started":"2022-12-06T10:11:56.799506Z","shell.execute_reply":"2022-12-06T10:11:56.810853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.normalize_total(cite_raw)\nsc.pp.log1p(cite_raw)\nsc.pp.scale(cite_raw, zero_center=False, max_value=10)","metadata":{"id":"vIu3K8xakXWT","outputId":"441565d9-bbb1-4daa-d84a-9329f13dc009","execution":{"iopub.status.busy":"2022-12-06T10:11:56.814180Z","iopub.execute_input":"2022-12-06T10:11:56.814985Z","iopub.status.idle":"2022-12-06T10:13:05.492465Z","shell.execute_reply.started":"2022-12-06T10:11:56.814937Z","shell.execute_reply":"2022-12-06T10:13:05.490586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.tl.pca(cite_raw, n_comps=50)","metadata":{"id":"k8OOQ4yJkXWT","execution":{"iopub.status.busy":"2022-12-06T10:13:05.494492Z","iopub.execute_input":"2022-12-06T10:13:05.495827Z","iopub.status.idle":"2022-12-06T10:14:29.440642Z","shell.execute_reply.started":"2022-12-06T10:13:05.495769Z","shell.execute_reply":"2022-12-06T10:14:29.439385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.pca(cite_raw, color=[\"cell_type\", \"donor\", \"day\"])","metadata":{"id":"l6x0ERoNkXWT","execution":{"iopub.status.busy":"2022-12-06T10:14:29.442529Z","iopub.execute_input":"2022-12-06T10:14:29.442879Z","iopub.status.idle":"2022-12-06T10:14:32.944781Z","shell.execute_reply.started":"2022-12-06T10:14:29.442848Z","shell.execute_reply":"2022-12-06T10:14:32.943488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.neighbors(cite_raw, n_pcs=25, n_neighbors=20)\nsc.tl.umap(cite_raw, min_dist=0.2)\nsc.pl.umap(cite_raw, color=[\"cell_type\", \"donor\", \"day\"], frameon=False)","metadata":{"id":"osUtmb0LkXWT","execution":{"iopub.status.busy":"2022-12-06T10:14:32.946659Z","iopub.execute_input":"2022-12-06T10:14:32.947637Z","iopub.status.idle":"2022-12-06T10:17:14.688069Z","shell.execute_reply.started":"2022-12-06T10:14:32.947587Z","shell.execute_reply":"2022-12-06T10:17:14.685173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pearson residuals","metadata":{"id":"lDF3UR2NkXWU"}},{"cell_type":"code","source":"cite_raw_train = sc.read_h5ad(FP_CITE_RAW_TRAIN_INPUTS)\ncite_raw_test = sc.read_h5ad(FP_CITE_RAW_TEST_INPUTS)\ncite_raw = ad.concat([cite_raw_train, cite_raw_test], merge=\"same\")\ndel cite_raw_train\ndel cite_raw_test\ngc.collect()\nannotate(cite_raw, metadata, f_donors = [\"13176\", ])\ncite_raw.layers[\"counts\"] = cite_raw.X.copy()\ncite_raw","metadata":{"execution":{"iopub.status.busy":"2022-12-06T18:10:05.954886Z","iopub.execute_input":"2022-12-06T18:10:05.955352Z","iopub.status.idle":"2022-12-06T18:11:16.329739Z","shell.execute_reply.started":"2022-12-06T18:10:05.955314Z","shell.execute_reply":"2022-12-06T18:11:16.328507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.experimental.pp.highly_variable_genes(cite_raw, theta=100/14, n_top_genes=N_top, batch_key=\"donor\")","metadata":{"id":"no7MkaGrkXWU","outputId":"d34bfe9c-d2d6-45a3-c793-db704b97685e","execution":{"iopub.status.busy":"2022-12-06T18:11:16.331838Z","iopub.execute_input":"2022-12-06T18:11:16.332192Z","iopub.status.idle":"2022-12-06T18:13:15.073034Z","shell.execute_reply.started":"2022-12-06T18:11:16.332162Z","shell.execute_reply":"2022-12-06T18:13:15.071731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.experimental.pp.normalize_pearson_residuals_pca(cite_raw, theta=100/14, n_comps=50)","metadata":{"id":"G0CN-OJpkXWU","outputId":"9638f7c6-e18d-47b5-8f4d-b1f629a4f479","execution":{"iopub.status.busy":"2022-12-06T18:13:15.074481Z","iopub.execute_input":"2022-12-06T18:13:15.074867Z","iopub.status.idle":"2022-12-06T18:14:00.283355Z","shell.execute_reply.started":"2022-12-06T18:13:15.074833Z","shell.execute_reply":"2022-12-06T18:14:00.282183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.pca(cite_raw, color=[\"cell_type\", \"donor\", \"day\", \"n_genes_counts\"])","metadata":{"id":"Y__lAaDhkXWV","outputId":"909ed3b3-1b5a-4ced-c125-0d8430553078","execution":{"iopub.status.busy":"2022-12-06T18:14:00.286197Z","iopub.execute_input":"2022-12-06T18:14:00.286694Z","iopub.status.idle":"2022-12-06T18:14:04.683597Z","shell.execute_reply.started":"2022-12-06T18:14:00.286659Z","shell.execute_reply":"2022-12-06T18:14:04.682305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hvg_pearson_residuals = cite_raw.var[cite_raw.var.highly_variable_intersection].index\nsave_hvgs(f\"HVG_cite_train_test_pearson_residuals_donor_{N_top}.txt\", hvg_pearson_residuals)\nlen(hvg_pearson_residuals)","metadata":{"id":"f3_O4ystkXWV","execution":{"iopub.status.busy":"2022-12-06T10:52:54.991566Z","iopub.execute_input":"2022-12-06T10:52:54.993940Z","iopub.status.idle":"2022-12-06T10:52:55.030238Z","shell.execute_reply.started":"2022-12-06T10:52:54.993641Z","shell.execute_reply":"2022-12-06T10:52:55.028118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pp.neighbors(cite_raw, n_pcs=15, n_neighbors=20)\nsc.tl.umap(cite_raw, min_dist=0.2)\nsc.pl.umap(cite_raw, color=[\"cell_type\", \"donor\", \"day\"], frameon=False)","metadata":{"id":"RoHeyzWAkXWV","execution":{"iopub.status.busy":"2022-12-06T18:14:04.685212Z","iopub.execute_input":"2022-12-06T18:14:04.685657Z","iopub.status.idle":"2022-12-06T18:17:44.947108Z","shell.execute_reply.started":"2022-12-06T18:14:04.685616Z","shell.execute_reply":"2022-12-06T18:17:44.944656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_raw.obs[\"MkP\"] = (cite_raw.obs[\"cell_type\"] == \"MkP\").astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:01:27.929904Z","iopub.execute_input":"2022-12-06T11:01:27.930413Z","iopub.status.idle":"2022-12-06T11:01:27.993771Z","shell.execute_reply.started":"2022-12-06T11:01:27.930366Z","shell.execute_reply":"2022-12-06T11:01:27.992219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.umap(cite_raw, color=[\"MkP\", \"day\"], frameon=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:01:52.700040Z","iopub.execute_input":"2022-12-06T11:01:52.700597Z","iopub.status.idle":"2022-12-06T11:01:55.015253Z","shell.execute_reply.started":"2022-12-06T11:01:52.700551Z","shell.execute_reply":"2022-12-06T11:01:55.013347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### by non-zero count","metadata":{"id":"X1uPLqWmkXWW"}},{"cell_type":"code","source":"sc.pl.umap(cite_raw, color=\"n_genes_counts\", frameon=False)","metadata":{"id":"ifT2nFmukXWW","execution":{"iopub.status.busy":"2022-12-06T11:03:39.613029Z","iopub.execute_input":"2022-12-06T11:03:39.613564Z","iopub.status.idle":"2022-12-06T11:03:40.490860Z","shell.execute_reply.started":"2022-12-06T11:03:39.613524Z","shell.execute_reply":"2022-12-06T11:03:40.489418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.umap(cite_raw, color=\"read_depth\", frameon=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T11:03:42.746968Z","iopub.execute_input":"2022-12-06T11:03:42.747444Z","iopub.status.idle":"2022-12-06T11:03:43.677459Z","shell.execute_reply.started":"2022-12-06T11:03:42.747405Z","shell.execute_reply":"2022-12-06T11:03:43.675932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### color by proliferation genes","metadata":{"id":"TN-OH2_tkXWX"}},{"cell_type":"code","source":"G1S_genes_Tirosh = ['ENSG00000100297', 'ENSG00000132646', 'ENSG00000176890', 'ENSG00000168496', 'ENSG00000073111', 'ENSG00000104738', 'ENSG00000167325', 'ENSG00000076248', 'ENSG00000131153', 'ENSG00000076003', 'ENSG00000144354', 'ENSG00000143476', 'ENSG00000198056', 'ENSG00000276043', 'ENSG00000151725', 'ENSG00000119969', 'ENSG00000049541', 'ENSG00000117748', 'ENSG00000132780', 'ENSG00000111247', 'ENSG00000112312', 'ENSG00000092470', 'ENSG00000163950', 'ENSG00000175305', 'ENSG00000012963', 'ENSG00000077514', 'ENSG00000095002', 'ENSG00000156802', 'ENSG00000051180', 'ENSG00000171848', 'ENSG00000093009', 'ENSG00000094804', 'ENSG00000174371', 'ENSG00000075131', 'ENSG00000136982', 'ENSG00000197299', 'ENSG00000118412', 'ENSG00000162607', 'ENSG00000092853', 'ENSG00000101868', 'ENSG00000159259', 'ENSG00000136492', 'ENSG00000129173']\nG2M_genes_Tirosh = ['ENSG00000164104', 'ENSG00000170312', 'ENSG00000137804', 'ENSG00000175063', 'ENSG00000089685', 'ENSG00000088325', 'ENSG00000131747', 'ENSG00000080986', 'ENSG00000123975', 'ENSG00000143228', 'ENSG00000173207', 'ENSG00000148773', 'ENSG00000120802', 'ENSG00000117724', 'ENSG00000013810', 'ENSG00000129195', 'ENSG00000113810', 'ENSG00000157456', 'ENSG00000169607', 'ENSG00000136108', 'ENSG00000178999', 'ENSG00000169679', 'ENSG00000138160', 'ENSG00000143401', 'ENSG00000188229', 'ENSG00000075218', 'ENSG00000138182', 'ENSG00000123485', 'ENSG00000111665', 'ENSG00000189159', 'ENSG00000117399', 'ENSG00000112742', 'ENSG00000158402', 'ENSG00000142945', 'ENSG00000100401', 'ENSG00000010292', 'ENSG00000126787', 'ENSG00000184661', 'ENSG00000134690', 'ENSG00000114346', 'ENSG00000137807', 'ENSG00000072571', 'ENSG00000087586', 'ENSG00000134222', 'ENSG00000011426', 'ENSG00000143815', 'ENSG00000175216', 'ENSG00000138778', 'ENSG00000102974', 'ENSG00000117650', 'ENSG00000092140', 'ENSG00000139354', 'ENSG00000094916', 'ENSG00000115163']","metadata":{"id":"2WHBY3vBkXWY","execution":{"iopub.status.busy":"2022-12-06T11:05:01.133276Z","iopub.execute_input":"2022-12-06T11:05:01.133795Z","iopub.status.idle":"2022-12-06T11:05:01.145113Z","shell.execute_reply.started":"2022-12-06T11:05:01.133745Z","shell.execute_reply":"2022-12-06T11:05:01.143494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_raw.var_names = [ t.split('_')[0] for t in cite_raw.var_names]","metadata":{"id":"Otp9mGTXkXWZ","execution":{"iopub.status.busy":"2022-12-06T11:05:32.796525Z","iopub.execute_input":"2022-12-06T11:05:32.796970Z","iopub.status.idle":"2022-12-06T11:05:32.818399Z","shell.execute_reply.started":"2022-12-06T11:05:32.796937Z","shell.execute_reply":"2022-12-06T11:05:32.816757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_raw.obs[\"G1S_genes_Tirosh_mean\"] = cite_raw[:,G1S_genes_Tirosh].X.sum(axis=1).A1 / len(G1S_genes_Tirosh)\ncite_raw.obs[\"G2M_genes_Tirosh_mean\"] = cite_raw[:,G2M_genes_Tirosh].X.sum(axis=1).A1 / len(G2M_genes_Tirosh)","metadata":{"id":"02mSzf9ekXWZ","execution":{"iopub.status.busy":"2022-12-06T11:05:35.398038Z","iopub.execute_input":"2022-12-06T11:05:35.398579Z","iopub.status.idle":"2022-12-06T11:05:38.475736Z","shell.execute_reply.started":"2022-12-06T11:05:35.398524Z","shell.execute_reply":"2022-12-06T11:05:38.474606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.umap(cite_raw, color=[\"cell_type\", \"G1S_genes_Tirosh_mean\", \"G2M_genes_Tirosh_mean\", \"n_genes_counts\", \"day\", \"donor\"], frameon=False, ncols=3)","metadata":{"id":"OoCO_ArckXWZ","execution":{"iopub.status.busy":"2022-12-06T11:05:42.786406Z","iopub.execute_input":"2022-12-06T11:05:42.786841Z","iopub.status.idle":"2022-12-06T11:05:48.621666Z","shell.execute_reply.started":"2022-12-06T11:05:42.786804Z","shell.execute_reply":"2022-12-06T11:05:48.619699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### color by proteins","metadata":{"id":"SF9PK5VrkXWZ"}},{"cell_type":"code","source":"cite_target = sc.read_h5ad(FP_CITE_TRAIN_TARGETS_ann)","metadata":{"id":"3QQuXFnvkXWa","execution":{"iopub.status.busy":"2022-12-06T18:17:44.949310Z","iopub.execute_input":"2022-12-06T18:17:44.950150Z","iopub.status.idle":"2022-12-06T18:17:46.646850Z","shell.execute_reply.started":"2022-12-06T18:17:44.950099Z","shell.execute_reply":"2022-12-06T18:17:46.645790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_raw_w_target = cite_raw[cite_target.obs_names].copy()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T18:17:46.648131Z","iopub.execute_input":"2022-12-06T18:17:46.648477Z","iopub.status.idle":"2022-12-06T18:17:54.612045Z","shell.execute_reply.started":"2022-12-06T18:17:46.648445Z","shell.execute_reply":"2022-12-06T18:17:54.610776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ndel cite_raw\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T18:17:54.613713Z","iopub.execute_input":"2022-12-06T18:17:54.614070Z","iopub.status.idle":"2022-12-06T18:17:55.002327Z","shell.execute_reply.started":"2022-12-06T18:17:54.614038Z","shell.execute_reply":"2022-12-06T18:17:55.001041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_raw_w_target.obs.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-06T18:17:55.004589Z","iopub.execute_input":"2022-12-06T18:17:55.004989Z","iopub.status.idle":"2022-12-06T18:17:55.024876Z","shell.execute_reply.started":"2022-12-06T18:17:55.004952Z","shell.execute_reply":"2022-12-06T18:17:55.023615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prots_df = pd.DataFrame(cite_target.X.A, columns=cite_target.var_names, index=cite_raw_w_target.obs.index)\ncite_raw_w_target.obs = pd.concat([cite_raw_w_target.obs, prots_df], axis=1)\n\ncite_raw_w_target.obs.head()","metadata":{"id":"msR4VnW3kXWa","execution":{"iopub.status.busy":"2022-12-06T18:17:55.027566Z","iopub.execute_input":"2022-12-06T18:17:55.028416Z","iopub.status.idle":"2022-12-06T18:17:55.284628Z","shell.execute_reply.started":"2022-12-06T18:17:55.028374Z","shell.execute_reply":"2022-12-06T18:17:55.283343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.set_figure_params(dpi=40)","metadata":{"id":"yC13Th-hkXWa","execution":{"iopub.status.busy":"2022-12-06T18:17:55.286076Z","iopub.execute_input":"2022-12-06T18:17:55.286974Z","iopub.status.idle":"2022-12-06T18:17:55.293010Z","shell.execute_reply.started":"2022-12-06T18:17:55.286934Z","shell.execute_reply":"2022-12-06T18:17:55.291721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prot_names = list(cite_target.var_names)\nprot_names.sort()","metadata":{"id":"1RryxqX0kXWb","execution":{"iopub.status.busy":"2022-12-06T18:17:55.294824Z","iopub.execute_input":"2022-12-06T18:17:55.295565Z","iopub.status.idle":"2022-12-06T18:17:55.303637Z","shell.execute_reply.started":"2022-12-06T18:17:55.295496Z","shell.execute_reply":"2022-12-06T18:17:55.302786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sc.pl.umap(cite_raw_w_target, color=prot_names, frameon=False, ncols=12)","metadata":{"id":"tm0ys81QkXWb","execution":{"iopub.status.busy":"2022-12-06T18:17:55.305348Z","iopub.execute_input":"2022-12-06T18:17:55.305708Z","iopub.status.idle":"2022-12-06T18:18:49.551772Z","shell.execute_reply.started":"2022-12-06T18:17:55.305673Z","shell.execute_reply":"2022-12-06T18:18:49.548948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save all HVGs to dataframe\n(precomputed output is available in the research-project-01-around-multimodal-singlecell dataset)","metadata":{}},{"cell_type":"code","source":"print(len(hvg_seurat), len(hvg_cr), len(hvg_seurat_v3), len(hvg_pearson_residuals))","metadata":{"execution":{"iopub.status.busy":"2022-12-04T11:28:23.201149Z","iopub.execute_input":"2022-12-04T11:28:23.201603Z","iopub.status.idle":"2022-12-04T11:28:23.209236Z","shell.execute_reply.started":"2022-12-04T11:28:23.201563Z","shell.execute_reply":"2022-12-04T11:28:23.207771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_methods = {\"seurat\": hvg_seurat, \"cell_ranger\": hvg_cr, \"seurat_v3\": hvg_seurat_v3, \"pearson_residuals\": hvg_pearson_residuals}\ndf = pd.DataFrame.from_dict(all_methods, orient='index')\ndf = df.transpose()\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T11:28:23.211459Z","iopub.execute_input":"2022-12-04T11:28:23.211927Z","iopub.status.idle":"2022-12-04T11:28:23.352703Z","shell.execute_reply.started":"2022-12-04T11:28:23.211887Z","shell.execute_reply":"2022-12-04T11:28:23.351330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.tail()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T11:28:23.354448Z","iopub.execute_input":"2022-12-04T11:28:23.354844Z","iopub.status.idle":"2022-12-04T11:28:23.369334Z","shell.execute_reply.started":"2022-12-04T11:28:23.354808Z","shell.execute_reply":"2022-12-04T11:28:23.367999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(f\"HVG_cite_train_test_donor_{N_top}.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T11:28:49.628006Z","iopub.execute_input":"2022-12-04T11:28:49.628513Z","iopub.status.idle":"2022-12-04T11:28:49.649298Z","shell.execute_reply.started":"2022-12-04T11:28:49.628475Z","shell.execute_reply":"2022-12-04T11:28:49.647884Z"},"trusted":true},"execution_count":null,"outputs":[]}]}