{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Some features can be dropped safely\n\nI toyed around a bit and found couple features, which are a subset of other features. This means that a feature has their values covered by other feature and therefore can be safely dropped.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\n\ncols = ['customer_ID','D_103','D_107','D_139','D_145']\nz = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet')[cols].fillna(-1)\nw = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet')[cols].fillna(-1)\nx = pd.concat([z,w])","metadata":{"execution":{"iopub.status.busy":"2022-07-20T12:45:50.528124Z","iopub.execute_input":"2022-07-20T12:45:50.528507Z","iopub.status.idle":"2022-07-20T12:46:35.764244Z","shell.execute_reply.started":"2022-07-20T12:45:50.528479Z","shell.execute_reply":"2022-07-20T12:46:35.758448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### drop D_103\n\n```\n- D_103 =-1 ~ D_107 =-1\n- D_103 = 0 ~ D_107 = 0\n- D_103 = 1 ~ D_107 > 0\n```","metadata":{}},{"cell_type":"code","source":"x.groupby(['D_103','D_107'])['customer_ID'].agg('count').reset_index(name='n')","metadata":{"execution":{"iopub.status.busy":"2022-07-20T12:46:35.769483Z","iopub.execute_input":"2022-07-20T12:46:35.769865Z","iopub.status.idle":"2022-07-20T12:46:37.175886Z","shell.execute_reply.started":"2022-07-20T12:46:35.769833Z","shell.execute_reply":"2022-07-20T12:46:37.174717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### drop D_139\n\n```\n- D_139 =-1 ~ D_145 =-1\n- D_139 = 0 ~ D_145 = 0\n- D_139 = 1 ~ D_145 > 0\n```","metadata":{}},{"cell_type":"code","source":"x.groupby(['D_139','D_145'])['customer_ID'].agg('count').reset_index(name='n')","metadata":{"execution":{"iopub.status.busy":"2022-07-20T12:46:37.177313Z","iopub.execute_input":"2022-07-20T12:46:37.177634Z","iopub.status.idle":"2022-07-20T12:46:38.741068Z","shell.execute_reply.started":"2022-07-20T12:46:37.177605Z","shell.execute_reply":"2022-07-20T12:46:38.739560Z"},"trusted":true},"execution_count":null,"outputs":[]}]}