{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Share how to compress and split multiome data sets.\nTest data is only cell_id required for submission.\n\nデータの分割と圧縮方法を共有致します。\nテストデータは提出に必要なcell_idのみで作成しています。","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport gc\nfrom tqdm import tqdm","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##train file handling\n# for i in tqdm(range(6)):\n#   n=i*20000\n#   if i==0:\n#     train_multi=pd.read_hdf('../input/open-problems-multimodal/train_multi_inputs.h5',start=i,stop=n+20000)\n#     train_multi.drop(drop_id,inplace=True)\n#     train_multi.reset_index(inplace=True)\n#     train_multi.to_feather(f'./train_multi_inputs_{i}.ftr')\n#     del test_multi\n#     gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluation_file=pd.read_csv('../input/open-problems-multimodal/evaluation_ids.csv')\n# evaluation_file.reset_index(inplace=True)\n# evaluation_file=evaluation_file.tail(58931360)#multiome_test_data\n# ev_id=evaluation_file['cell_id']\n# del evaluation_file\n# gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##test file handling\n# for i in tqdm(range(3)):\n#   n=i*20000\n#   if i==0:\n#     test_multi=pd.read_hdf('../input/open-problems-multimodal/test_multi_inputs.h5',start=i,stop=n+20000)\n#     test_index=train_multi.index\n#     drop_id=[c for c in test_index if c not in ev_id]\n#     test_multi.drop(drop_id,inplace=True)\n#     test_multi.reset_index(inplace=True)\n#     test_multi.to_feather(f'./test_multi_inputs_{i}.ftr')\n#     del test_multi\n#     gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Enjoy Competition!","metadata":{}}]}