{"metadata":{"kernelspec":{"name":"ir","display_name":"R","language":"R"},"language_info":{"name":"R","codemirror_mode":"r","pygments_lexer":"r","mimetype":"text/x-r-source","file_extension":".r","version":"4.0.5"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"rhdf5 is used to load h5 files","metadata":{}},{"cell_type":"code","source":"if (!require(\"BiocManager\", quietly = TRUE))\n    install.packages(\"BiocManager\")\n\nBiocManager::install(\"rhdf5\")\n","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:04:52.090724Z","iopub.execute_input":"2022-08-20T21:04:52.093304Z","iopub.status.idle":"2022-08-20T21:05:11.766366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"library(rhdf5)\nlibrary(Matrix)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:11:13.746341Z","iopub.execute_input":"2022-08-20T21:11:13.74811Z","iopub.status.idle":"2022-08-20T21:11:15.093239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"setwd('/kaggle/input/open-problems-multimodal')\ngetwd()\nlist.files()","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:06:27.362991Z","iopub.execute_input":"2022-08-20T21:06:27.365399Z","iopub.status.idle":"2022-08-20T21:06:27.399277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta = read.csv(\"metadata.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:06:40.800359Z","iopub.execute_input":"2022-08-20T21:06:40.802147Z","iopub.status.idle":"2022-08-20T21:06:41.900932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta[1:5,]","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:06:42.106723Z","iopub.execute_input":"2022-08-20T21:06:42.108425Z","iopub.status.idle":"2022-08-20T21:06:42.135649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"meta column distribution","metadata":{}},{"cell_type":"code","source":"setwd(\"/kaggle/working\") \npar(mfrow=c(2,2))\nbarplot(table(meta$day), main = \"Day distribution\")\nbarplot(table(meta$donor),main = \"Donor distribution\")\nbarplot(table(meta$cell_type), main = \"cell type distribution\")\nbarplot(table(meta$technology), main = \"Tech distribution\")\n","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:06:44.946192Z","iopub.execute_input":"2022-08-20T21:06:44.947808Z","iopub.status.idle":"2022-08-20T21:06:45.396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"day, donor, distribuion almost matched. cell distribution is not even which maybe from the tissue.","metadata":{}},{"cell_type":"code","source":"dim(meta)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:06:47.971008Z","iopub.execute_input":"2022-08-20T21:06:47.972531Z","iopub.status.idle":"2022-08-20T21:06:47.98702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"h5ls is used to list the data in the file","metadata":{}},{"cell_type":"code","source":"h5ls(\"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:06:49.722056Z","iopub.execute_input":"2022-08-20T21:06:49.723573Z","iopub.status.idle":"2022-08-20T21:06:50.443296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"genes <- h5read(\"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\", \"/train_cite_inputs/axis0\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:06:52.212841Z","iopub.execute_input":"2022-08-20T21:06:52.214512Z","iopub.status.idle":"2022-08-20T21:06:52.268068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"head(genes)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:06:53.624781Z","iopub.execute_input":"2022-08-20T21:06:53.626466Z","iopub.status.idle":"2022-08-20T21:06:53.643029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cite_cells <- h5read(\"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\", \"/train_cite_inputs/axis1\")\nhead(train_cite_cells)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:06:55.815317Z","iopub.execute_input":"2022-08-20T21:06:55.817099Z","iopub.status.idle":"2022-08-20T21:06:55.890512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cite_input<-h5read(\"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\", \"/train_cite_inputs/block0_values\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:06:58.871244Z","iopub.execute_input":"2022-08-20T21:06:58.873106Z","iopub.status.idle":"2022-08-20T21:07:52.284131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cite_input <- Matrix(train_cite_input, sparse = TRUE) \ngc()","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:11:21.478545Z","iopub.execute_input":"2022-08-20T21:11:21.480378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = meta[match(train_cite_cells,meta$cell_id),]","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:03:41.10084Z","iopub.execute_input":"2022-08-20T21:03:41.102499Z","iopub.status.idle":"2022-08-20T21:03:41.146924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample(which(tmp$day == 2),100)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:04:10.21415Z","iopub.execute_input":"2022-08-20T21:04:10.215799Z","iopub.status.idle":"2022-08-20T21:04:10.233523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"boxplot(train_cite_input[,sample(which(tmp$day == 2),100)],train_cite_input[,sample(which(tmp$day == 3),100)],train_cite_input[,sample(which(tmp$day == 4),100)])","metadata":{"execution":{"iopub.status.busy":"2022-08-20T21:04:18.372675Z","iopub.execute_input":"2022-08-20T21:04:18.374514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dim(train_cite_input)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:55:24.176569Z","iopub.execute_input":"2022-08-20T20:55:24.178139Z","iopub.status.idle":"2022-08-20T20:55:24.194011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rm(train_cite_input)\ngc()","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:05.309415Z","iopub.execute_input":"2022-08-20T20:48:05.310923Z","iopub.status.idle":"2022-08-20T20:48:06.121478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h5ls(\"/kaggle/input/open-problems-multimodal/train_cite_targets.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:10.257728Z","iopub.execute_input":"2022-08-20T20:48:10.259332Z","iopub.status.idle":"2022-08-20T20:48:10.313409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_target_cells <- h5read(\"/kaggle/input/open-problems-multimodal/train_cite_targets.h5\", \"/train_cite_targets/axis1\")\ntrain_target_protein <- h5read(\"/kaggle/input/open-problems-multimodal/train_cite_targets.h5\", \"/train_cite_targets/axis0\")\nhead(train_target_cells)\nhead(train_target_protein)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:12.60833Z","iopub.execute_input":"2022-08-20T20:48:12.609897Z","iopub.status.idle":"2022-08-20T20:48:12.689671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train_target_cells and train_cite_cells are the same","metadata":{}},{"cell_type":"code","source":"length(which(train_target_cells!=train_cite_cells))","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:15.463575Z","iopub.execute_input":"2022-08-20T20:48:15.466163Z","iopub.status.idle":"2022-08-20T20:48:15.484341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train cite cell distribuiont across different meta characters","metadata":{}},{"cell_type":"code","source":"tmp = meta[match(train_cite_cells,meta$cell_id),]\nsetwd(\"/kaggle/working\") \npar(mfrow=c(2,2))\nbarplot(table(tmp$day)/nrow(tmp), main = \"Day distribution\")\nbarplot(table(tmp$donor)/nrow(tmp),main = \"Donor distribution\")\nbarplot(table(tmp$cell_type)/nrow(tmp), main = \"cell type distribution\")\nbarplot(table(tmp$technology)/nrow(tmp), main = \"Tech distribution\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:17.729966Z","iopub.execute_input":"2022-08-20T20:48:17.73183Z","iopub.status.idle":"2022-08-20T20:48:17.920508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmpx = table(tmp$cell_type)/nrow(tmp)\ntmpy = table(meta$cell_type[which(meta$cell_type != \"hidden\")])/length(which(meta$cell_type != \"hidden\"))\nbarplot(rbind(tmpx,tmpy),beside=T,col=c(\"blue\", \"red\")) \nlegend(\"topright\", legend = c(\"train_cite_cell\",\"meta_cell\"),fill = c(\"blue\", \"red\"))","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:21.098078Z","iopub.execute_input":"2022-08-20T20:48:21.099987Z","iopub.status.idle":"2022-08-20T20:48:21.25484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h5ls(\"/kaggle/input/open-problems-multimodal/train_multi_inputs.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:24.461589Z","iopub.execute_input":"2022-08-20T20:48:24.463186Z","iopub.status.idle":"2022-08-20T20:48:29.934175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train_multi and train_multi_target have the same cells","metadata":{}},{"cell_type":"code","source":"train_multi_cells <- h5read(\"/kaggle/input/open-problems-multimodal/train_multi_inputs.h5\", \"/train_multi_inputs/axis1\")\ntrain_multi_target_cells <- h5read(\"/kaggle/input/open-problems-multimodal/train_multi_targets.h5\", \"/train_multi_targets/axis1\")\nlength(which(train_multi_cells!=train_multi_target_cells))","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:33.852011Z","iopub.execute_input":"2022-08-20T20:48:33.853702Z","iopub.status.idle":"2022-08-20T20:48:34.005519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train multi cells distribution across meta characters","metadata":{}},{"cell_type":"markdown","source":"train multi cell distribution","metadata":{}},{"cell_type":"code","source":"tmp = meta[match(train_multi_cells,meta$cell_id),]\npar(mfrow=c(2,2))\nbarplot(table(tmp$day)/nrow(tmp), main = \"Day distribution\")\nbarplot(table(tmp$donor)/nrow(tmp),main = \"Donor distribution\")\nbarplot(table(tmp$cell_type)/nrow(tmp), main = \"cell type distribution\")\nbarplot(table(tmp$technology)/nrow(tmp), main = \"Tech distribution\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:36.243958Z","iopub.execute_input":"2022-08-20T20:48:36.245627Z","iopub.status.idle":"2022-08-20T20:48:36.432086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmpz = table(tmp$cell_type)/nrow(tmp)\nbarplot(rbind(tmpx,tmpy,tmpz),beside=T,col=c(\"blue\", \"red\", \"green\")) \nlegend(\"topright\", legend = c(\"train_cite_cell\",\"meta_cell\", \"train_multi_cell\"),fill = c(\"blue\", \"red\", \"green\"))","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:39.104635Z","iopub.execute_input":"2022-08-20T20:48:39.106336Z","iopub.status.idle":"2022-08-20T20:48:39.205909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h5ls(\"/kaggle/input/open-problems-multimodal/train_multi_targets.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:43.584176Z","iopub.execute_input":"2022-08-20T20:48:43.585896Z","iopub.status.idle":"2022-08-20T20:48:44.5757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h5ls(\"/kaggle/input/open-problems-multimodal/test_multi_inputs.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:47.016167Z","iopub.execute_input":"2022-08-20T20:48:47.017773Z","iopub.status.idle":"2022-08-20T20:48:49.353743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"h5ls(\"/kaggle/input/open-problems-multimodal/test_cite_inputs.h5\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:51.663943Z","iopub.execute_input":"2022-08-20T20:48:51.665492Z","iopub.status.idle":"2022-08-20T20:48:52.084259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_cite_cells<-h5read(\"/kaggle/input/open-problems-multimodal/test_cite_inputs.h5\", \"/test_cite_inputs/axis1\")\ntest_multi_cells<-h5read(\"/kaggle/input/open-problems-multimodal/test_multi_inputs.h5\", \"/test_multi_inputs/axis1\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:55.426583Z","iopub.execute_input":"2022-08-20T20:48:55.428337Z","iopub.status.idle":"2022-08-20T20:48:55.541755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"test multi cells distribution across different characters","metadata":{}},{"cell_type":"code","source":"tmp = meta[match(test_multi_cells,meta$cell_id),]\npar(mfrow=c(2,2))\nbarplot(table(tmp$day)/nrow(tmp), main = \"Day distribution\")\nbarplot(table(tmp$donor)/nrow(tmp),main = \"Donor distribution\")\nbarplot(table(tmp$cell_type)/nrow(tmp), main = \"cell type distribution\")\nbarplot(table(tmp$technology)/nrow(tmp), main = \"Tech distribution\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:48:58.022126Z","iopub.execute_input":"2022-08-20T20:48:58.023647Z","iopub.status.idle":"2022-08-20T20:48:58.160743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"test cite cells distribution across different characters","metadata":{}},{"cell_type":"code","source":"tmp = meta[match(test_cite_cells,meta$cell_id),]\npar(mfrow=c(2,2))\nbarplot(table(tmp$day)/nrow(tmp), main = \"Day distribution\")\nbarplot(table(tmp$donor)/nrow(tmp),main = \"Donor distribution\")\nbarplot(table(tmp$cell_type)/nrow(tmp), main = \"cell type distribution\")\nbarplot(table(tmp$technology)/nrow(tmp), main = \"Tech distribution\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:49:00.794219Z","iopub.execute_input":"2022-08-20T20:49:00.795751Z","iopub.status.idle":"2022-08-20T20:49:00.933187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmpu = table(tmp$cell_type)/nrow(tmp)\nbarplot(rbind(tmpx,tmpy,tmpz, tmpu),beside=T,col=c(\"blue\", \"red\", \"green\", \"orange\")) \nlegend(\"topright\", legend = c(\"train_cite_cell\",\"meta_cell\", \"train_multi_cell\", \"test_cite_cell\"),fill = c(\"blue\", \"red\", \"green\", \"orange\"))","metadata":{"execution":{"iopub.status.busy":"2022-08-20T20:49:03.664045Z","iopub.execute_input":"2022-08-20T20:49:03.665636Z","iopub.status.idle":"2022-08-20T20:49:03.761414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"day, and donor distribution are very different between test and control. cell type distibution matches better in train_cite, train_multi, test_cite and meta. We need to correction the batch effect, especially from day and donor","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}