{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What is about ?\n\nExample to work with huge h5 files - data is backed on disk, but one can work with it\n**as if it is loaded into memory**. \n\n(That is one of the key features for h5 files https://en.wikipedia.org/wiki/Hierarchical_Data_Format). \n\nUse \"import h5py\"  (and additional \"import hdf5plugin\" - which resolved  some unexpected error appeared using just h5py for that particular data file). \n\nOther examples for (mostly) single cell RNA sequencing data - see ARCHS4 collection of datasets from GEO database: \nhttps://www.kaggle.com/code/alexandervc/archs4-extractsave-datasets-by-gse-and-keyword\n\nPS \n\n    For h5ad files which are typical in scRNA-seq analysis by scanpy one can also do similar trick,\n    adata = sc.read(fn, backed='r' )\n    see example: \nhttps://www.kaggle.com/code/alexandervc/scanpy-process-huge-files-backing-them-on-disk\n\n\n(c) Alexander Chervov, for kaggle competition: \"Open Problems - Multimodal Single-Cell Integration\"\n\nPSPS\n\nOne might find 50+ single cell RNA sequencing datasets on kaggle:\nhttps://www.kaggle.com/search?q=scRNA-seq+in%3Adatasets\n\nAnd hundreds notebooks  to analyse the cell cycle for them. \nThat analysis is summarized in our recent paper: https://arxiv.org/abs/2208.05229\n\n    Computational challenges of cell cycle analysis using single cell transcriptomics\n    Alexander Chervov, Andrei Zinovyev\n\nIt might be of some interest for the competition also. \nFor cell cycle information see: https://en.wikipedia.org/wiki/Cell_cycle\n\nPSPSPS\n\nPlease consider/support the idea to create \"Bionformatics community\" on Kaggle:\nhttps://www.kaggle.com/general/203136\n\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-18T18:24:30.751097Z","iopub.execute_input":"2022-08-18T18:24:30.751499Z","iopub.status.idle":"2022-08-18T18:24:30.759740Z","shell.execute_reply.started":"2022-08-18T18:24:30.751463Z","shell.execute_reply":"2022-08-18T18:24:30.758871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Briefly - one needs only that:\n\n","metadata":{}},{"cell_type":"code","source":"import h5py\n!pip install hdf5plugin~=2.0 # https://forum.hdfgroup.org/t/cant-open-directory-usr-local-hdf5-lib-plugin/9738/4\nimport hdf5plugin\n\nprint('Look on Features: ')\nfilename = '/kaggle/input/open-problems-multimodal/train_multi_inputs.h5'\nf2 = h5py.File(filename,'r')#, mode)\n\n\nprint(f2.keys() )\nprint(f2['train_multi_inputs'].keys())\nprint('Values of feature matrix:')\nprint( f2['train_multi_inputs']['block0_values'][:100,:100] )\nprint('fragment of a DNA ids:')\nprint( f2['train_multi_inputs']['axis0'][:5] )\nprint('Cell Ids (seems to me):')\nprint( f2['train_multi_inputs']['axis1'][:5] )\n\nprint()\nprint('Look on TARGETs: ')\nfilename = '/kaggle/input/open-problems-multimodal/train_multi_targets.h5'\nf = h5py.File(filename,'r')#, mode)\nprint('Values matrix to predict:')\nprint( f['train_multi_targets']['block0_values'][:100,:100] )\nprint('fragment of a DNA Ids:')\nprint( f['train_multi_targets']['axis0'][:5] )\nprint('Cell Ids (seems to me):')\nprint( f['train_multi_targets']['axis1'][:5] )\n","metadata":{"execution":{"iopub.status.busy":"2022-08-18T19:34:11.157528Z","iopub.execute_input":"2022-08-18T19:34:11.158051Z","iopub.status.idle":"2022-08-18T19:34:11.284214Z","shell.execute_reply.started":"2022-08-18T19:34:11.158015Z","shell.execute_reply":"2022-08-18T19:34:11.283071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Long tutorial way","metadata":{}},{"cell_type":"code","source":"import h5py","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:24:56.272076Z","iopub.execute_input":"2022-08-18T18:24:56.272477Z","iopub.status.idle":"2022-08-18T18:24:56.277502Z","shell.execute_reply.started":"2022-08-18T18:24:56.272442Z","shell.execute_reply":"2022-08-18T18:24:56.276451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename = '/kaggle/input/open-problems-multimodal/train_multi_targets.h5'\nf = h5py.File(filename,'r')#, mode)","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:25:18.822862Z","iopub.execute_input":"2022-08-18T18:25:18.823246Z","iopub.status.idle":"2022-08-18T18:25:18.837673Z","shell.execute_reply.started":"2022-08-18T18:25:18.823214Z","shell.execute_reply":"2022-08-18T18:25:18.836708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f.keys()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:25:29.229309Z","iopub.execute_input":"2022-08-18T18:25:29.229724Z","iopub.status.idle":"2022-08-18T18:25:29.241089Z","shell.execute_reply.started":"2022-08-18T18:25:29.229685Z","shell.execute_reply":"2022-08-18T18:25:29.240183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f['train_multi_targets']","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:25:37.976984Z","iopub.execute_input":"2022-08-18T18:25:37.977352Z","iopub.status.idle":"2022-08-18T18:25:37.991859Z","shell.execute_reply.started":"2022-08-18T18:25:37.977320Z","shell.execute_reply":"2022-08-18T18:25:37.990712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f['train_multi_targets'].keys()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:25:49.772946Z","iopub.execute_input":"2022-08-18T18:25:49.773987Z","iopub.status.idle":"2022-08-18T18:25:49.781899Z","shell.execute_reply.started":"2022-08-18T18:25:49.773944Z","shell.execute_reply":"2022-08-18T18:25:49.780705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in list(f['train_multi_targets'].keys()):\n    print(k)\n    print(f['train_multi_targets'][k])\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:26:58.626770Z","iopub.execute_input":"2022-08-18T18:26:58.627173Z","iopub.status.idle":"2022-08-18T18:26:58.636221Z","shell.execute_reply.started":"2022-08-18T18:26:58.627139Z","shell.execute_reply":"2022-08-18T18:26:58.635141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in list(f['train_multi_targets'].keys()):\n    print(k)\n    #for k2 in list(f['train_multi_targets'][k].keys() ):\n    print(type(f['train_multi_targets'][k]))\n    print('shape:', (f['train_multi_targets'][k].shape))\n    print('Dir:', dir(f['train_multi_targets'][k]))\n    \n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:40:21.549228Z","iopub.execute_input":"2022-08-18T18:40:21.549613Z","iopub.status.idle":"2022-08-18T18:40:21.559771Z","shell.execute_reply.started":"2022-08-18T18:40:21.549580Z","shell.execute_reply":"2022-08-18T18:40:21.558888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f['train_multi_targets']['block0_items'][0]","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:46:23.989191Z","iopub.execute_input":"2022-08-18T18:46:23.989608Z","iopub.status.idle":"2022-08-18T18:46:24.047873Z","shell.execute_reply.started":"2022-08-18T18:46:23.989571Z","shell.execute_reply":"2022-08-18T18:46:24.046373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install hdf5plugin~=2.0 # https://forum.hdfgroup.org/t/cant-open-directory-usr-local-hdf5-lib-plugin/9738/4","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:48:03.550332Z","iopub.execute_input":"2022-08-18T18:48:03.550711Z","iopub.status.idle":"2022-08-18T18:48:16.933893Z","shell.execute_reply.started":"2022-08-18T18:48:03.550679Z","shell.execute_reply":"2022-08-18T18:48:16.932579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install hdf5plugin~=2.0 # https://forum.hdfgroup.org/t/cant-open-directory-usr-local-hdf5-lib-plugin/9738/4\nimport h5py\nimport hdf5plugin\n","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:48:22.807992Z","iopub.execute_input":"2022-08-18T18:48:22.808802Z","iopub.status.idle":"2022-08-18T18:48:22.824374Z","shell.execute_reply.started":"2022-08-18T18:48:22.808747Z","shell.execute_reply":"2022-08-18T18:48:22.823517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f['train_multi_targets']['block0_items'][0]","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:48:35.047391Z","iopub.execute_input":"2022-08-18T18:48:35.047840Z","iopub.status.idle":"2022-08-18T18:48:35.058891Z","shell.execute_reply.started":"2022-08-18T18:48:35.047789Z","shell.execute_reply":"2022-08-18T18:48:35.057882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(5):\n    print('i=',i, f['train_multi_targets']['axis0'][i],  f['train_multi_targets']['axis1'][i],  f['train_multi_targets']['block0_items'][i] )","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:51:00.501234Z","iopub.execute_input":"2022-08-18T18:51:00.502019Z","iopub.status.idle":"2022-08-18T18:51:00.518747Z","shell.execute_reply.started":"2022-08-18T18:51:00.501956Z","shell.execute_reply":"2022-08-18T18:51:00.517779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f['train_multi_targets']['axis0'][:5]","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:51:37.742229Z","iopub.execute_input":"2022-08-18T18:51:37.742625Z","iopub.status.idle":"2022-08-18T18:51:37.751056Z","shell.execute_reply.started":"2022-08-18T18:51:37.742593Z","shell.execute_reply":"2022-08-18T18:51:37.749892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f['train_multi_targets']['axis1'][:5]","metadata":{"execution":{"iopub.status.busy":"2022-08-18T19:21:31.059901Z","iopub.execute_input":"2022-08-18T19:21:31.060306Z","iopub.status.idle":"2022-08-18T19:21:31.070478Z","shell.execute_reply.started":"2022-08-18T19:21:31.060274Z","shell.execute_reply":"2022-08-18T19:21:31.069401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f['train_multi_targets']['block0_values'][:3,:3]","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:52:27.857225Z","iopub.execute_input":"2022-08-18T18:52:27.857614Z","iopub.status.idle":"2022-08-18T18:52:27.887709Z","shell.execute_reply.started":"2022-08-18T18:52:27.857584Z","shell.execute_reply":"2022-08-18T18:52:27.886425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f['train_multi_targets']['block0_values'][:100,:100]","metadata":{"execution":{"iopub.status.busy":"2022-08-18T18:52:42.158322Z","iopub.execute_input":"2022-08-18T18:52:42.158727Z","iopub.status.idle":"2022-08-18T18:52:42.250697Z","shell.execute_reply.started":"2022-08-18T18:52:42.158690Z","shell.execute_reply":"2022-08-18T18:52:42.249609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}