{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install episcanpy","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-29T08:16:41.171745Z","iopub.execute_input":"2022-08-29T08:16:41.172254Z","iopub.status.idle":"2022-08-29T08:17:18.413126Z","shell.execute_reply.started":"2022-08-29T08:16:41.172159Z","shell.execute_reply":"2022-08-29T08:17:18.412009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q tables","metadata":{"execution":{"iopub.status.busy":"2022-08-29T08:17:18.415304Z","iopub.execute_input":"2022-08-29T08:17:18.415667Z","iopub.status.idle":"2022-08-29T08:17:30.627203Z","shell.execute_reply.started":"2022-08-29T08:17:18.41563Z","shell.execute_reply":"2022-08-29T08:17:30.625739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\n\nimport scanpy as sc\nimport anndata as ad\nimport numpy as np\nimport pandas as pd\nimport episcanpy.api as epi\n\nfrom tqdm.auto import tqdm\nfrom multiprocessing import Pool","metadata":{"execution":{"iopub.status.busy":"2022-08-29T08:17:30.629796Z","iopub.execute_input":"2022-08-29T08:17:30.630289Z","iopub.status.idle":"2022-08-29T08:17:34.026704Z","shell.execute_reply.started":"2022-08-29T08:17:30.630231Z","shell.execute_reply":"2022-08-29T08:17:34.025631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train multi data contains 105942 cells\ndef ch_str(x):\n    x = x.replace(':', '_')\n    x = x.replace('-', '_')\n    return x\n\nclass get_geneactivity():\n    def __init__(self, state='train'):\n        self.state = state\n    def __call__(self, start):\n        start *= 1000\n        atac_data = pd.read_hdf(f'../input/open-problems-multimodal/{self.state}_multi_inputs.h5', start=start, stop=start+1000)\n        atac_data.columns = [ch_str(x) for x in atac_data.columns]\n        atac_data = ad.AnnData(atac_data)\n        atac_data = epi.tl.geneactivity(atac_data,\n                              '../input/grch38-gtf/gencode.v32.primary_assembly.annotation.gtf',\n                              upstream=2000)\n        atac_data.var['gene_id'] = [x.split('.')[0] for x in atac_data.var['gene_id']]\n        atac_data = pd.DataFrame(atac_data.X, index=atac_data.obs_names, columns=atac_data.var['gene_id'])\n        return atac_data","metadata":{"execution":{"iopub.status.busy":"2022-08-29T08:17:34.029049Z","iopub.execute_input":"2022-08-29T08:17:34.029399Z","iopub.status.idle":"2022-08-29T08:17:34.04046Z","shell.execute_reply.started":"2022-08-29T08:17:34.029368Z","shell.execute_reply":"2022-08-29T08:17:34.038872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Deal with Test Data","metadata":{"execution":{"iopub.status.busy":"2022-08-29T07:47:23.066682Z","iopub.execute_input":"2022-08-29T07:47:23.067154Z","iopub.status.idle":"2022-08-29T07:47:26.764689Z","shell.execute_reply.started":"2022-08-29T07:47:23.067114Z","shell.execute_reply":"2022-08-29T07:47:26.763247Z"}}},{"cell_type":"code","source":"# test multi data contains 55935 cells\nn_iter = 55935 // 1000 + 1\nprint(n_iter)","metadata":{"execution":{"iopub.status.busy":"2022-08-29T07:47:30.239239Z","iopub.execute_input":"2022-08-29T07:47:30.239687Z","iopub.status.idle":"2022-08-29T07:47:30.247893Z","shell.execute_reply.started":"2022-08-29T07:47:30.239643Z","shell.execute_reply":"2022-08-29T07:47:30.246806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"func = get_geneactivity('test')\nwith Pool(processes=4) as pool:\n    r = pool.map(func, range(n_iter))\nr = pd.concat(r, axis=0)\nr.to_hdf('test_multi_geneactivity.h5', 'key', mode='w')","metadata":{"execution":{"iopub.status.busy":"2022-08-29T07:41:30.378349Z","iopub.execute_input":"2022-08-29T07:41:30.378751Z","iopub.status.idle":"2022-08-29T07:41:30.38634Z","shell.execute_reply.started":"2022-08-29T07:41:30.378717Z","shell.execute_reply":"2022-08-29T07:41:30.385222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}