{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Intro\nQuick exploration of row/col counts, filesizes and sparsity for you to think about CPU/RAM provisioning.","metadata":{}},{"cell_type":"code","source":"!pip install tables","metadata":{"execution":{"iopub.status.busy":"2022-08-20T06:55:14.084562Z","iopub.execute_input":"2022-08-20T06:55:14.085212Z","iopub.status.idle":"2022-08-20T06:55:27.850122Z","shell.execute_reply.started":"2022-08-20T06:55:14.085115Z","shell.execute_reply":"2022-08-20T06:55:27.848164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-20T06:55:27.852964Z","iopub.execute_input":"2022-08-20T06:55:27.853467Z","iopub.status.idle":"2022-08-20T06:55:27.860374Z","shell.execute_reply.started":"2022-08-20T06:55:27.853421Z","shell.execute_reply":"2022-08-20T06:55:27.858942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FP = {\n    'trci': '../input/open-problems-multimodal/train_cite_inputs.h5',\n    'trct': '../input/open-problems-multimodal/train_cite_targets.h5',\n    'trmi': '../input/open-problems-multimodal/train_multi_inputs.h5',\n    'trmt': '../input/open-problems-multimodal/train_multi_targets.h5',\n    'teci': '../input/open-problems-multimodal/test_cite_inputs.h5',\n    'temi': '../input/open-problems-multimodal/test_multi_inputs.h5',\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-20T06:55:27.862803Z","iopub.execute_input":"2022-08-20T06:55:27.863353Z","iopub.status.idle":"2022-08-20T06:55:27.872026Z","shell.execute_reply.started":"2022-08-20T06:55:27.863296Z","shell.execute_reply":"2022-08-20T06:55:27.870735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Row and column counts for all .h5 files","metadata":{}},{"cell_type":"code","source":"for key, fn in FP.items():\n    hs = pd.HDFStore(fn, mode='r')\n    print(hs.info())","metadata":{"execution":{"iopub.status.busy":"2022-08-20T06:55:27.876637Z","iopub.execute_input":"2022-08-20T06:55:27.877296Z","iopub.status.idle":"2022-08-20T06:55:28.523918Z","shell.execute_reply.started":"2022-08-20T06:55:27.877255Z","shell.execute_reply":"2022-08-20T06:55:28.522302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nrows = {\n    'trci': 70988,\n    'trct': 70988,\n    'trmi': 105942,\n    'trmt': 105942,\n    'teci': 48663,\n    'temi': 55935,\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-20T06:55:28.525779Z","iopub.execute_input":"2022-08-20T06:55:28.526481Z","iopub.status.idle":"2022-08-20T06:55:28.531572Z","shell.execute_reply.started":"2022-08-20T06:55:28.526437Z","shell.execute_reply":"2022-08-20T06:55:28.530369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Uncompressed filesizes (approx., if read by pandas)","metadata":{}},{"cell_type":"code","source":"%%time\nfor key in FP.keys():\n    df = pd.read_hdf(FP[key], start=0, stop=100)\n    mem_gb = (df.memory_usage().sum() / 1e9) * (nrows[key] / 100)\n    print(f'{FP[key].split(\"/\")[-1] :23}: {mem_gb :.3f} GB')\nprint('\\n')","metadata":{"execution":{"iopub.status.busy":"2022-08-20T06:55:28.533147Z","iopub.execute_input":"2022-08-20T06:55:28.533670Z","iopub.status.idle":"2022-08-20T06:56:02.675212Z","shell.execute_reply.started":"2022-08-20T06:55:28.533511Z","shell.execute_reply":"2022-08-20T06:56:02.673957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sparsity (proportion of values close to zero)","metadata":{}},{"cell_type":"code","source":"%%time\nfn = FP['trci']\nsp = pd.DataFrame()\nfor i in range(120):\n    df = pd.read_hdf(fn, start=i*1000, stop=i*1000+1000)\n    if len(df) < 1:\n        break\n    sparsity = (df < 1e-3).sum().sum() / df.count().sum()\n    \n    sp.loc[i, 'n'] = len(df)\n    sp.loc[i, 'sparsity'] = sparsity\n    \nagg_sparsity_trci = (sp.n * sp.sparsity).sum() / sp.n.sum()\nprint(f'Sparsity of {fn.split(\"/\")[-1]}: {agg_sparsity_trci}')\nprint('\\n')","metadata":{"execution":{"iopub.status.busy":"2022-08-20T06:56:02.677109Z","iopub.execute_input":"2022-08-20T06:56:02.677501Z","iopub.status.idle":"2022-08-20T06:57:12.706251Z","shell.execute_reply.started":"2022-08-20T06:56:02.677466Z","shell.execute_reply":"2022-08-20T06:57:12.704791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfn = FP['trmi']\nsp = pd.DataFrame()\nfor i in range(120):\n    df = pd.read_hdf(fn, start=i*1000, stop=i*1000+1000)\n    if len(df) < 1:\n        break\n    sparsity = (df < 1e-3).sum().sum() / df.count().sum()\n    \n    sp.loc[i, 'n'] = len(df)\n    sp.loc[i, 'sparsity'] = sparsity\n    \nagg_sparsity_trmi = (sp.n * sp.sparsity).sum() / sp.n.sum()\nprint(f'Sparsity of {fn.split(\"/\")[-1]}: {agg_sparsity_trmi}')\nprint('\\n')","metadata":{"execution":{"iopub.status.busy":"2022-08-20T06:57:12.708623Z","iopub.execute_input":"2022-08-20T06:57:12.709248Z","iopub.status.idle":"2022-08-20T07:08:39.772815Z","shell.execute_reply.started":"2022-08-20T06:57:12.709192Z","shell.execute_reply":"2022-08-20T07:08:39.770859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}