{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nimport time\nimport warnings\nimport pickle\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sn\nimport missingno as msno\n\nimport gc\n\nwarnings.filterwarnings('ignore')\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-06-18T05:26:32.515676Z","iopub.execute_input":"2022-06-18T05:26:32.51612Z","iopub.status.idle":"2022-06-18T05:26:32.52443Z","shell.execute_reply.started":"2022-06-18T05:26:32.516085Z","shell.execute_reply":"2022-06-18T05:26:32.523283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display settings\npd.set_option('display.max_rows', 200)\npd.set_option('display.max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2022-06-18T05:26:34.915887Z","iopub.execute_input":"2022-06-18T05:26:34.916326Z","iopub.status.idle":"2022-06-18T05:26:34.92133Z","shell.execute_reply.started":"2022-06-18T05:26:34.916289Z","shell.execute_reply":"2022-06-18T05:26:34.920064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_PATH\nTRAIN_DATA_PATH = \"../input/amex-default-prediction/train_data.csv\"\nTEST_DATA_PATH = \"../input/amex-default-prediction/test_data.csv\"\nTRAIN_LABELS_PATH = \"../input/amex-default-prediction/train_labels.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-06-18T05:26:37.137112Z","iopub.execute_input":"2022-06-18T05:26:37.137688Z","iopub.status.idle":"2022-06-18T05:26:37.143835Z","shell.execute_reply.started":"2022-06-18T05:26:37.13764Z","shell.execute_reply":"2022-06-18T05:26:37.142355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_labels\ntrain_labels_df = pd.read_csv(TRAIN_LABELS_PATH)\nprint(train_labels_df.shape)\nprint(train_labels_df['target'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-06-18T05:27:28.653279Z","iopub.execute_input":"2022-06-18T05:27:28.653717Z","iopub.status.idle":"2022-06-18T05:27:29.271954Z","shell.execute_reply.started":"2022-06-18T05:27:28.653686Z","shell.execute_reply":"2022-06-18T05:27:29.270844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_spl = pd.read_csv(TRAIN_DATA_PATH, nrows=10)\nall_cols = train_df_spl.columns.tolist()\nprint(train_df_spl.columns.tolist())","metadata":{"execution":{"iopub.status.busy":"2022-06-18T05:36:55.688105Z","iopub.execute_input":"2022-06-18T05:36:55.688658Z","iopub.status.idle":"2022-06-18T05:36:55.715383Z","shell.execute_reply.started":"2022-06-18T05:36:55.688613Z","shell.execute_reply":"2022-06-18T05:36:55.714103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# columns category group list for chunk loading\ncols01 = [elem for elem in all_cols[1:] if 'D' in elem ][0:32]\nprint(\"cols01: \" + str(len(cols01)))\nprint(cols01)\n\ncols02 = [elem for elem in all_cols[1:] if 'D' in elem ][32:64]\nprint(\"cols02: \" + str(len(cols02)))\nprint(cols02)\n\ncols03 = [elem for elem in all_cols[1:] if 'D' in elem ][64:96]\nprint(\"cols03: \" + str(len(cols03)))\nprint(cols03)\n\ncols04 = [elem for elem in all_cols[1:] if 'B' in elem ]\nprint(\"cols04: \" + str(len(cols04)))\nprint(cols04)\n\ncols05 = [elem for elem in all_cols[1:] if 'R' in elem ]\nprint(\"cols05: \" + str(len(cols05)))\nprint(cols05)\n\ncols06 = [elem for elem in all_cols[1:] if ('S' in elem) or ('P' in elem) ]\nprint(\"cols06: \" + str(len(cols06)))\nprint(cols06)","metadata":{"execution":{"iopub.status.busy":"2022-06-18T05:57:29.763028Z","iopub.execute_input":"2022-06-18T05:57:29.763525Z","iopub.status.idle":"2022-06-18T05:57:29.776261Z","shell.execute_reply.started":"2022-06-18T05:57:29.763473Z","shell.execute_reply":"2022-06-18T05:57:29.774998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# load data for NA count (train/test)\n\nusecol_list = [cols01,cols02,cols03,cols04,cols05,cols06]\nna_summary_tab = pd.DataFrame()\n\nfor col_ in usecol_list:\n    chunksize = 1000000\n    # train_df = train_df_iter.__next__()\n    train_df_iter = pd.read_csv(TRAIN_DATA_PATH, chunksize=chunksize, usecols=[\"customer_ID\"] + col_)\n    train_df_ = pd.DataFrame()\n    for i_chunk, chunk in enumerate(train_df_iter):\n        train_df_ = pd.concat([train_df_, chunk])\n        print(train_df_.shape)\n    del chunk\n\n    # msno.matrix(train_df_, figsize=(20,5), color=(0.2,0.2,0.5))\n\n    train_nan_tab = pd.DataFrame(train_df_.isnull().sum(), columns=['train_NA_count'])\n    train_nap_tab = pd.DataFrame(train_df_.isnull().sum()/len(train_df_)*100, columns=['train_NA_pct'])\n\n    del train_df_\n    gc.collect()\n\n\n    chunksize = 1000000\n    # teat_df = test_df_iter.__next__()\n    test_df_iter = pd.read_csv(TEST_DATA_PATH, chunksize=chunksize, usecols=[\"customer_ID\"] + col_)\n    test_df_ = pd.DataFrame()\n    for i_chunk, chunk in enumerate(test_df_iter):\n        test_df_ = pd.concat([test_df_, chunk])\n        print(test_df_.shape)\n    del chunk\n\n    # msno.matrix(test_df_, figsize=(20,5), color=(0.5,0.2,0.2))\n\n    test_nan_tab = pd.DataFrame(test_df_.isnull().sum(), columns=['test_NA_count'])\n    test_nap_tab = pd.DataFrame(test_df_.isnull().sum()/len(test_df_)*100, columns=['test_NA_pct'])\n\n    del test_df_\n    gc.collect()\n\n\n    nan_tab = pd.merge(train_nan_tab, test_nan_tab,\n                       left_index=True, right_index=True, how='outer')\n    nap_tab = pd.merge(train_nap_tab, test_nap_tab,\n                       left_index=True, right_index=True, how='outer')\n\n    na_tab = pd.merge(nan_tab, nap_tab,\n                      left_index=True, right_index=True, how='outer')\n    gc.collect()\n\n    na_summary_tab = pd.concat([na_summary_tab, na_tab])\n\n# save na summary (train/test)\nna_summary_tab.to_csv('na_summary_table.csv', index=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-15T10:01:56.66994Z","iopub.execute_input":"2022-06-15T10:01:56.670248Z","iopub.status.idle":"2022-06-15T10:16:26.906304Z","shell.execute_reply.started":"2022-06-15T10:01:56.670223Z","shell.execute_reply":"2022-06-15T10:16:26.905094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"na_summary_tab.drop(index='customer_ID')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"na_summary_tab.query('train_NA_count == 0 and test_NA_count ==0').drop(index='customer_ID')","metadata":{},"execution_count":null,"outputs":[]}]}