{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Check Multimodal Data Constant Features","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport scipy.sparse\nimport copy\nfrom collections import defaultdict, Counter","metadata":{"execution":{"iopub.status.busy":"2022-09-06T11:35:03.451197Z","iopub.execute_input":"2022-09-06T11:35:03.451868Z","iopub.status.idle":"2022-09-06T11:35:03.569231Z","shell.execute_reply.started":"2022-09-06T11:35:03.451776Z","shell.execute_reply":"2022-09-06T11:35:03.568302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install tables","metadata":{"execution":{"iopub.status.busy":"2022-09-06T11:35:03.570900Z","iopub.execute_input":"2022-09-06T11:35:03.571483Z","iopub.status.idle":"2022-09-06T11:35:18.001531Z","shell.execute_reply.started":"2022-09-06T11:35:03.571447Z","shell.execute_reply":"2022-09-06T11:35:18.000309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_constant_columns(filename, chunksize=2500, checksize=105942):\n    start = 0\n    total_rows = 0\n    \n    no_need_to_check_features = []\n    features_dict = defaultdict(lambda: 0)\n    \n    while True:\n        df_chunk = pd.read_hdf(filename, start=start, stop=start+chunksize)\n        \n        if len(df_chunk) == 0:\n            break\n        zero_num_series = (df_chunk==0).sum(axis=0).to_dict()\n        features_dict = dict(Counter(features_dict) + Counter(zero_num_series))\n            \n        total_rows += len(df_chunk)\n        print(total_rows)\n        if len(df_chunk) < chunksize: \n            del df_chunk\n            break\n            \n        del df_chunk\n        start += chunksize\n    \n    retain_features = [k for k,v in features_dict.items() if v!=checksize]\n    constant_features = [k for k,v in features_dict.items() if v==checksize]\n    print(f\"There are {len(retain_features)} features retained, \")\n    print(constant_features)\n    \n    return retain_features, constant_features","metadata":{"execution":{"iopub.status.busy":"2022-09-06T11:35:18.003300Z","iopub.execute_input":"2022-09-06T11:35:18.003687Z","iopub.status.idle":"2022-09-06T11:35:18.015099Z","shell.execute_reply.started":"2022-09-06T11:35:18.003651Z","shell.execute_reply":"2022-09-06T11:35:18.013859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"retain_features, constant_features = check_constant_columns('../input/open-problems-multimodal/train_multi_inputs.h5', chunksize=2500, checksize=105942)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T11:35:18.018756Z","iopub.execute_input":"2022-09-06T11:35:18.019163Z","iopub.status.idle":"2022-09-06T11:42:51.612146Z","shell.execute_reply.started":"2022-09-06T11:35:18.019127Z","shell.execute_reply":"2022-09-06T11:42:51.610848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"retain_features_test, constant_features_test = check_constant_columns('../input/open-problems-multimodal/test_multi_inputs.h5', chunksize=2500, checksize=55935)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T11:42:51.614338Z","iopub.execute_input":"2022-09-06T11:42:51.614811Z","iopub.status.idle":"2022-09-06T11:46:58.211533Z","shell.execute_reply.started":"2022-09-06T11:42:51.614767Z","shell.execute_reply":"2022-09-06T11:46:58.210400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_constant_u = set(constant_features).union(set(constant_features_test))\nall_constant_i = set(constant_features).intersection(set(constant_features_test))","metadata":{"execution":{"iopub.status.busy":"2022-09-06T11:46:58.213018Z","iopub.execute_input":"2022-09-06T11:46:58.213396Z","iopub.status.idle":"2022-09-06T11:46:58.219278Z","shell.execute_reply.started":"2022-09-06T11:46:58.213363Z","shell.execute_reply":"2022-09-06T11:46:58.218064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(all_constant_u), len(all_constant_i)","metadata":{"execution":{"iopub.status.busy":"2022-09-06T11:46:58.220634Z","iopub.execute_input":"2022-09-06T11:46:58.220946Z","iopub.status.idle":"2022-09-06T11:46:58.236825Z","shell.execute_reply.started":"2022-09-06T11:46:58.220917Z","shell.execute_reply":"2022-09-06T11:46:58.235923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_constant_u","metadata":{"execution":{"iopub.status.busy":"2022-09-06T11:46:58.237872Z","iopub.execute_input":"2022-09-06T11:46:58.238553Z","iopub.status.idle":"2022-09-06T11:46:58.250920Z","shell.execute_reply.started":"2022-09-06T11:46:58.238521Z","shell.execute_reply":"2022-09-06T11:46:58.249740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_constant_i","metadata":{"execution":{"iopub.status.busy":"2022-09-06T11:46:58.252841Z","iopub.execute_input":"2022-09-06T11:46:58.253254Z","iopub.status.idle":"2022-09-06T11:46:58.263846Z","shell.execute_reply.started":"2022-09-06T11:46:58.253197Z","shell.execute_reply":"2022-09-06T11:46:58.262737Z"},"trusted":true},"execution_count":null,"outputs":[]}]}