{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What is about ? \n\nThe dataframe created and saved to csv - with big list of genes ids - Ensembl, Entrez, Symbol and a bit of other information.\n\nGenes Ids collected from the several public scRNA-seq dataset. \n\n\n59K+ genes processed. Human only. \n\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:05.563896Z","iopub.execute_input":"2022-12-23T16:26:05.564422Z","iopub.status.idle":"2022-12-23T16:26:05.673323Z","shell.execute_reply.started":"2022-12-23T16:26:05.564316Z","shell.execute_reply":"2022-12-23T16:26:05.671974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = \"/kaggle/input/open-problems-multimodal/\"\nFP_CELL_METADATA = os.path.join(DATA_DIR,\"metadata.csv\")\n\nFP_CITE_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_cite_inputs.h5\")\nFP_CITE_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_cite_targets.h5\")\nFP_CITE_TEST_INPUTS = os.path.join(DATA_DIR,\"test_cite_inputs.h5\")\n\nFP_MULTIOME_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_multi_inputs.h5\")\nFP_MULTIOME_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_multi_targets.h5\")\nFP_MULTIOME_TEST_INPUTS = os.path.join(DATA_DIR,\"test_multi_inputs.h5\")\n\nFP_SUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")\nFP_EVALUATION_IDS = os.path.join(DATA_DIR,\"evaluation_ids.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:05.893645Z","iopub.execute_input":"2022-12-23T16:26:05.894621Z","iopub.status.idle":"2022-12-23T16:26:05.904375Z","shell.execute_reply.started":"2022-12-23T16:26:05.894551Z","shell.execute_reply":"2022-12-23T16:26:05.903035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#If you see a urllib warning running this cell, go to \"Settings\" on the right hand side, \n#and turn on internet. Note, you need to be phone verified.\n!pip install --quiet tables","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:05.906413Z","iopub.execute_input":"2022-12-23T16:26:05.907045Z","iopub.status.idle":"2022-12-23T16:26:20.893913Z","shell.execute_reply.started":"2022-12-23T16:26:05.907001Z","shell.execute_reply":"2022-12-23T16:26:20.892359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data from multiome part of  NIPS22 competition ","metadata":{}},{"cell_type":"code","source":"df_multi_train_y = pd.read_hdf(FP_MULTIOME_TRAIN_TARGETS, start=0, stop=10)\ndf_multi_train_y","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:20.896790Z","iopub.execute_input":"2022-12-23T16:26:20.898638Z","iopub.status.idle":"2022-12-23T16:26:21.132776Z","shell.execute_reply.started":"2022-12-23T16:26:20.898575Z","shell.execute_reply":"2022-12-23T16:26:21.131535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_genes = list(df_multi_train_y.columns)\nprint(len(list_genes), list_genes[:10])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:21.134440Z","iopub.execute_input":"2022-12-23T16:26:21.134831Z","iopub.status.idle":"2022-12-23T16:26:21.142917Z","shell.execute_reply.started":"2022-12-23T16:26:21.134795Z","shell.execute_reply":"2022-12-23T16:26:21.141548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data from CITE-seq part of  NIPS22 competition ","metadata":{}},{"cell_type":"code","source":"df_cite_train_x = pd.read_hdf(FP_CITE_TRAIN_INPUTS, stop = 5)\ndf_cite_train_x ","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:21.146483Z","iopub.execute_input":"2022-12-23T16:26:21.146937Z","iopub.status.idle":"2022-12-23T16:26:21.277012Z","shell.execute_reply.started":"2022-12-23T16:26:21.146897Z","shell.execute_reply":"2022-12-23T16:26:21.275596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = [t.split('_')[0] for t in df_cite_train_x.columns]\nprint(len(l), l[:10])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:21.278917Z","iopub.execute_input":"2022-12-23T16:26:21.279377Z","iopub.status.idle":"2022-12-23T16:26:21.294487Z","shell.execute_reply.started":"2022-12-23T16:26:21.279339Z","shell.execute_reply":"2022-12-23T16:26:21.293103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_genes = list( set(list_genes) | set(l)  )\nprint(len(list_genes), list_genes[:10])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:21.295997Z","iopub.execute_input":"2022-12-23T16:26:21.296891Z","iopub.status.idle":"2022-12-23T16:26:21.321508Z","shell.execute_reply.started":"2022-12-23T16:26:21.296849Z","shell.execute_reply":"2022-12-23T16:26:21.320358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data from NIPS21 competition ","metadata":{}},{"cell_type":"code","source":"!pip install scanpy\nimport scanpy as sc","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:21.325792Z","iopub.execute_input":"2022-12-23T16:26:21.326202Z","iopub.status.idle":"2022-12-23T16:26:39.411046Z","shell.execute_reply.started":"2022-12-23T16:26:21.326164Z","shell.execute_reply":"2022-12-23T16:26:39.409436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfn = '/kaggle/input/citeseqscrnaseqproteins-challenge-neurips2021/GSE194122_openproblems_neurips2021_cite_BMMC_processed.h5ad'\nadata = sc.read(fn)\nprint(adata)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:39.412963Z","iopub.execute_input":"2022-12-23T16:26:39.413383Z","iopub.status.idle":"2022-12-23T16:26:52.948092Z","shell.execute_reply.started":"2022-12-23T16:26:39.413335Z","shell.execute_reply":"2022-12-23T16:26:52.946731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = list( adata.var[ 'gene_id'][ adata.var[ 'gene_id'].notnull()]  )\nprint(len(l), l[:10])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:52.950649Z","iopub.execute_input":"2022-12-23T16:26:52.951174Z","iopub.status.idle":"2022-12-23T16:26:52.966905Z","shell.execute_reply.started":"2022-12-23T16:26:52.951124Z","shell.execute_reply":"2022-12-23T16:26:52.964318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_genes = list( set(list_genes) | set(l)  )\nprint(len(list_genes), list_genes[:10])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:52.969204Z","iopub.execute_input":"2022-12-23T16:26:52.970669Z","iopub.status.idle":"2022-12-23T16:26:53.059211Z","shell.execute_reply.started":"2022-12-23T16:26:52.970623Z","shell.execute_reply":"2022-12-23T16:26:53.057977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## And genes from sciPlex dataset","metadata":{}},{"cell_type":"code","source":"%%time\nfn = '/kaggle/input/genes-information/genes_human_58347_used_in_sciPlex2_brief_info_by_mygene_package.csv'\nd = pd.read_csv(fn )\nd","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:53.061192Z","iopub.execute_input":"2022-12-23T16:26:53.061749Z","iopub.status.idle":"2022-12-23T16:26:53.664840Z","shell.execute_reply.started":"2022-12-23T16:26:53.061692Z","shell.execute_reply":"2022-12-23T16:26:53.663490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = [t.split('.')[0] for t in d['ensembl id'] ]\nprint(len(l), l[:10])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:53.666711Z","iopub.execute_input":"2022-12-23T16:26:53.667221Z","iopub.status.idle":"2022-12-23T16:26:53.701567Z","shell.execute_reply.started":"2022-12-23T16:26:53.667172Z","shell.execute_reply":"2022-12-23T16:26:53.700740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_genes = list( set(list_genes) | set(l)  )\nprint(len(list_genes), list_genes[:10])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:53.705451Z","iopub.execute_input":"2022-12-23T16:26:53.706224Z","iopub.status.idle":"2022-12-23T16:26:53.737281Z","shell.execute_reply.started":"2022-12-23T16:26:53.706187Z","shell.execute_reply":"2022-12-23T16:26:53.735407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add another list of 46983 genes\n","metadata":{}},{"cell_type":"code","source":"fn = '/kaggle/input/genes-information/46983GenesInformations.csv'\nd = pd.read_csv(fn )\nd","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:53.739518Z","iopub.execute_input":"2022-12-23T16:26:53.740177Z","iopub.status.idle":"2022-12-23T16:26:54.017275Z","shell.execute_reply.started":"2022-12-23T16:26:53.740136Z","shell.execute_reply":"2022-12-23T16:26:54.016027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = [t.split('.')[0] for t in d['Ensembl'] ]\nprint(len(l), l[:10])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:54.019025Z","iopub.execute_input":"2022-12-23T16:26:54.019517Z","iopub.status.idle":"2022-12-23T16:26:54.042597Z","shell.execute_reply.started":"2022-12-23T16:26:54.019467Z","shell.execute_reply":"2022-12-23T16:26:54.041515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_genes = list( set(list_genes) | set(l)  )\nprint(len(list_genes), list_genes[:10])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:54.044307Z","iopub.execute_input":"2022-12-23T16:26:54.045418Z","iopub.status.idle":"2022-12-23T16:26:54.082647Z","shell.execute_reply.started":"2022-12-23T16:26:54.045379Z","shell.execute_reply":"2022-12-23T16:26:54.081028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Entrez Ids, etc by the mygene package","metadata":{}},{"cell_type":"code","source":"!pip install mygene\nimport mygene","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:26:54.084676Z","iopub.execute_input":"2022-12-23T16:26:54.085620Z","iopub.status.idle":"2022-12-23T16:27:06.299534Z","shell.execute_reply.started":"2022-12-23T16:26:54.085565Z","shell.execute_reply":"2022-12-23T16:27:06.297969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint(len(list_genes), list_genes[:5])\nmg = mygene.MyGeneInfo()\ng = mg.getgenes( list_genes,  fields=['symbol',  'alias',   'name',  'entrezgene', 'map_location', 'type_of_gene'  ], species='human'\n               , as_dataframe=True)# [:1000])\ng","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:27:06.303753Z","iopub.execute_input":"2022-12-23T16:27:06.304168Z","iopub.status.idle":"2022-12-23T16:29:45.525081Z","shell.execute_reply.started":"2022-12-23T16:27:06.304127Z","shell.execute_reply":"2022-12-23T16:29:45.523480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g2 = g.copy()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:29:45.526997Z","iopub.execute_input":"2022-12-23T16:29:45.527398Z","iopub.status.idle":"2022-12-23T16:29:45.570684Z","shell.execute_reply.started":"2022-12-23T16:29:45.527354Z","shell.execute_reply":"2022-12-23T16:29:45.569136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g2.index.name = 'Ensembl ID'\ng2 = g2.reset_index()\ng2","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:29:45.572474Z","iopub.execute_input":"2022-12-23T16:29:45.572874Z","iopub.status.idle":"2022-12-23T16:29:45.610543Z","shell.execute_reply.started":"2022-12-23T16:29:45.572836Z","shell.execute_reply":"2022-12-23T16:29:45.609448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g2 = g2.drop(['_id', '_version'], axis = 1)\ng2","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:29:45.612135Z","iopub.execute_input":"2022-12-23T16:29:45.612546Z","iopub.status.idle":"2022-12-23T16:29:45.673026Z","shell.execute_reply.started":"2022-12-23T16:29:45.612509Z","shell.execute_reply":"2022-12-23T16:29:45.671810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g2 = g2.drop(['notfound'], axis = 1)\ng2","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:29:45.675017Z","iopub.execute_input":"2022-12-23T16:29:45.675696Z","iopub.status.idle":"2022-12-23T16:29:45.706565Z","shell.execute_reply.started":"2022-12-23T16:29:45.675649Z","shell.execute_reply":"2022-12-23T16:29:45.704673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = g2.duplicated(subset = 'Ensembl ID', keep=False)# .sum()\ng2[m]","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:29:45.708350Z","iopub.execute_input":"2022-12-23T16:29:45.709480Z","iopub.status.idle":"2022-12-23T16:29:45.765688Z","shell.execute_reply.started":"2022-12-23T16:29:45.709424Z","shell.execute_reply":"2022-12-23T16:29:45.763909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g3 = g2.drop_duplicates('Ensembl ID' )\nprint(g2.shape, g3.shape)\ng3","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:29:45.767446Z","iopub.execute_input":"2022-12-23T16:29:45.767932Z","iopub.status.idle":"2022-12-23T16:29:45.837370Z","shell.execute_reply.started":"2022-12-23T16:29:45.767869Z","shell.execute_reply":"2022-12-23T16:29:45.836085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g3.index = range(len(g3))\ng3","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:29:45.839386Z","iopub.execute_input":"2022-12-23T16:29:45.840175Z","iopub.status.idle":"2022-12-23T16:29:45.861787Z","shell.execute_reply.started":"2022-12-23T16:29:45.840122Z","shell.execute_reply":"2022-12-23T16:29:45.860341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g3.head(50)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:29:45.863780Z","iopub.execute_input":"2022-12-23T16:29:45.864222Z","iopub.status.idle":"2022-12-23T16:29:45.904658Z","shell.execute_reply.started":"2022-12-23T16:29:45.864181Z","shell.execute_reply":"2022-12-23T16:29:45.903084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(g3['type_of_gene'].isnull().sum(), g3['type_of_gene'].notnull().sum() )\ng3['type_of_gene'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:29:45.906719Z","iopub.execute_input":"2022-12-23T16:29:45.908173Z","iopub.status.idle":"2022-12-23T16:29:45.950638Z","shell.execute_reply.started":"2022-12-23T16:29:45.908121Z","shell.execute_reply":"2022-12-23T16:29:45.949357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## There are some duplicated entrezgene ids \n\n- reason is not clear , \nmay be because Ensembl Ids sometimes become outdated, and while entrez - not, so may be old and new enseml - points to the same entrez ? \n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-22T14:37:36.057351Z","iopub.execute_input":"2022-12-22T14:37:36.057837Z","iopub.status.idle":"2022-12-22T14:37:36.064236Z","shell.execute_reply.started":"2022-12-22T14:37:36.057800Z","shell.execute_reply":"2022-12-22T14:37:36.062337Z"}}},{"cell_type":"code","source":"m0 = g3['entrezgene'].notnull()\nprint(m0.sum())\nm = g3[m0].duplicated(subset = 'entrezgene', keep=False)# .sum()\nprint(m.sum())\nprint(m)\ng3[m0][m].head(20)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:33:04.009828Z","iopub.execute_input":"2022-12-23T16:33:04.010296Z","iopub.status.idle":"2022-12-23T16:33:04.085876Z","shell.execute_reply.started":"2022-12-23T16:33:04.010221Z","shell.execute_reply":"2022-12-23T16:33:04.084988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Some symbols are duplicated \n\nmay be some of them - for not well-studied genes - many assigned something like \"Y_RNA\"","metadata":{}},{"cell_type":"code","source":"m0 = g3['symbol'].notnull()\nprint(m0.sum())\nm = g3[m0].duplicated(subset = 'symbol', keep=False)# .sum()\nprint(m.sum())\nprint(m)\ndisplay(g3[m0][m].head(20))\ng3[g3['symbol'] == 'CHN2']\n","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:34:10.679092Z","iopub.execute_input":"2022-12-23T16:34:10.679732Z","iopub.status.idle":"2022-12-23T16:34:10.786191Z","shell.execute_reply.started":"2022-12-23T16:34:10.679675Z","shell.execute_reply":"2022-12-23T16:34:10.784736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## look on CD** genes ","metadata":{}},{"cell_type":"code","source":"l = []\nfor i,t in enumerate( g3['alias'].values):\n    for j in range(10):  \n        if (\"'CD\"+str(j) in str(t))  : \n            l.append( (i,t ) )\n            break\n            \nprint(len(l), l[:10])        \n","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:47:29.517488Z","iopub.execute_input":"2022-12-23T16:47:29.518326Z","iopub.status.idle":"2022-12-23T16:47:29.949867Z","shell.execute_reply.started":"2022-12-23T16:47:29.518279Z","shell.execute_reply":"2022-12-23T16:47:29.948568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save to csv","metadata":{}},{"cell_type":"code","source":"g3.to_csv('genes_big_list_ensembl_symbol_entrez_etc.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-23T16:29:46.039523Z","iopub.execute_input":"2022-12-23T16:29:46.040369Z","iopub.status.idle":"2022-12-23T16:29:46.300560Z","shell.execute_reply.started":"2022-12-23T16:29:46.040328Z","shell.execute_reply":"2022-12-23T16:29:46.299316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}