{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-17T07:41:56.355759Z","iopub.execute_input":"2022-06-17T07:41:56.356197Z","iopub.status.idle":"2022-06-17T07:41:56.387792Z","shell.execute_reply.started":"2022-06-17T07:41:56.356114Z","shell.execute_reply":"2022-06-17T07:41:56.38684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Understanding the file size of one file\nfrom humanize import naturalsize\nfrom dask import dataframe as dd\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:42:01.092077Z","iopub.execute_input":"2022-06-17T07:42:01.092474Z","iopub.status.idle":"2022-06-17T07:42:01.929317Z","shell.execute_reply.started":"2022-06-17T07:42:01.092441Z","shell.execute_reply":"2022-06-17T07:42:01.928481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fn = \"../input/amex-default-prediction/train_data.csv\"\ntrain_df = dd.read_csv(fn)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:42:09.262739Z","iopub.execute_input":"2022-06-17T07:42:09.263153Z","iopub.status.idle":"2022-06-17T07:42:09.449161Z","shell.execute_reply.started":"2022-06-17T07:42:09.263119Z","shell.execute_reply":"2022-06-17T07:42:09.448061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nastat = train_df.isna().sum().compute()\nnastat.sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:42:17.956952Z","iopub.execute_input":"2022-06-17T07:42:17.957416Z","iopub.status.idle":"2022-06-17T07:45:20.843587Z","shell.execute_reply.started":"2022-06-17T07:42:17.95734Z","shell.execute_reply":"2022-06-17T07:45:20.842005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"delcols =list(nastat.sort_values(ascending=False)[:12].index)\nfor col in delcols:\n    del train_df[col]","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:45:31.340392Z","iopub.execute_input":"2022-06-17T07:45:31.340934Z","iopub.status.idle":"2022-06-17T07:45:31.68069Z","shell.execute_reply.started":"2022-06-17T07:45:31.34088Z","shell.execute_reply":"2022-06-17T07:45:31.679792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:45:57.892136Z","iopub.execute_input":"2022-06-17T07:45:57.892631Z","iopub.status.idle":"2022-06-17T07:45:58.142352Z","shell.execute_reply.started":"2022-06-17T07:45:57.892592Z","shell.execute_reply":"2022-06-17T07:45:58.141178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_columns =['B_30','B_31','B_38','D_63','D_64','D_66','D_68','D_87','D_114','D_116','D_117','D_120','D_126']\nnon_cat_columns = set(train_df.columns) - set(cat_columns) - {\"customer_ID\", \"S_2\"}\nfloat_cols = list(non_cat_columns)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:47:17.098451Z","iopub.execute_input":"2022-06-17T07:47:17.098942Z","iopub.status.idle":"2022-06-17T07:47:17.106234Z","shell.execute_reply.started":"2022-06-17T07:47:17.098903Z","shell.execute_reply":"2022-06-17T07:47:17.105161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg1 = train_df.groupby(\"customer_ID\")[float_cols].agg(['mean', 'std', 'min', 'max', 'last'])","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:47:20.187133Z","iopub.execute_input":"2022-06-17T07:47:20.188033Z","iopub.status.idle":"2022-06-17T07:47:21.980519Z","shell.execute_reply.started":"2022-06-17T07:47:20.187991Z","shell.execute_reply":"2022-06-17T07:47:21.979404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfa= agg1.compute()\ndfa.columns = ['_'.join(x) for x in dfa.columns]","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:48:18.350336Z","iopub.execute_input":"2022-06-17T07:48:18.350857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_columns.remove('D_87')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg2 = dd_df.groupby(\"customer_ID\")[cat_columns].agg(['count', 'last'])\ndfb = agg2.compute()\ndfb.columns = ['_'.join(x) for x in dfb.columns]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# dfcnt = train_df.nunique().compute()\n# catcol = dfcnt[dfcnt<20]\n# cat_columns = list(catcol.index)\ncat_columns =['B_30','B_31','B_38','D_63','D_64','D_66','D_68','D_87' ,'D_114','D_116','D_117','D_120','D_126'] # null values is too much and has been dropped!\nfor col in cat_columns:\n    train_df[col] = train_df[col].astype('int8')\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-17T06:45:55.848423Z","iopub.execute_input":"2022-06-17T06:45:55.848842Z","iopub.status.idle":"2022-06-17T06:45:56.505315Z","shell.execute_reply.started":"2022-06-17T06:45:55.848812Z","shell.execute_reply":"2022-06-17T06:45:56.504518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the Competition Data Description:\n\nD_* = Delinquency variables <br/>\nS_* = Spend variables <br/>\nP_* = Payment variables  <br/>\nB_* = Balance variables <br/>\nR_* = Risk variables  <br/>","metadata":{}},{"cell_type":"code","source":"# for col in cat_columns:\n#     print(f\"{col} Unique Values: {train_df[col].unique().compute()} \")\nnon_cat_columns = set(train_df.columns) - set(cat_columns) \n\nd_columns = [col for col in non_cat_columns if \"D_\" in col]\ns_columns = [col for col in non_cat_columns if \"S_\" in col]\np_columns = [col for col in non_cat_columns if \"P_\" in col]\nr_columns = [col for col in non_cat_columns if \"R_\" in col]\nb_columns = [col for col in non_cat_columns if \"B_\" in col]\n\n# float_cols = set([*d_columns, *s_columns, *p_columns, *r_columns, *b_columns]) \nnum_cols = list(non_cat_columns)","metadata":{"execution":{"iopub.status.busy":"2022-06-17T06:48:09.751883Z","iopub.execute_input":"2022-06-17T06:48:09.752288Z","iopub.status.idle":"2022-06-17T06:48:09.75993Z","shell.execute_reply.started":"2022-06-17T06:48:09.752254Z","shell.execute_reply":"2022-06-17T06:48:09.758769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[num_cols].fillna(0) # inplace=True","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:06:33.280263Z","iopub.execute_input":"2022-06-17T07:06:33.280679Z","iopub.status.idle":"2022-06-17T07:06:33.520518Z","shell.execute_reply.started":"2022-06-17T07:06:33.280647Z","shell.execute_reply":"2022-06-17T07:06:33.51946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_num_agg = train_df.groupby(\"customer_ID\")[num_cols].agg(['mean', 'min', 'max']).compute()","metadata":{"execution":{"iopub.status.busy":"2022-06-17T07:07:38.306775Z","iopub.execute_input":"2022-06-17T07:07:38.307294Z","iopub.status.idle":"2022-06-17T07:07:38.920995Z","shell.execute_reply.started":"2022-06-17T07:07:38.307262Z","shell.execute_reply":"2022-06-17T07:07:38.919339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]","metadata":{"execution":{"iopub.status.busy":"2022-06-17T06:20:27.079064Z","iopub.execute_input":"2022-06-17T06:20:27.080158Z","iopub.status.idle":"2022-06-17T06:20:27.125051Z","shell.execute_reply.started":"2022-06-17T06:20:27.080104Z","shell.execute_reply":"2022-06-17T06:20:27.12395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfN = test_num_agg.compute() ","metadata":{"execution":{"iopub.status.busy":"2022-06-17T06:43:41.902355Z","iopub.execute_input":"2022-06-17T06:43:41.902799Z","iopub.status.idle":"2022-06-17T06:43:45.41549Z","shell.execute_reply.started":"2022-06-17T06:43:41.902764Z","shell.execute_reply":"2022-06-17T06:43:45.413646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-13T05:16:51.957966Z","iopub.execute_input":"2022-06-13T05:16:51.958654Z","iopub.status.idle":"2022-06-13T05:16:51.978416Z","shell.execute_reply.started":"2022-06-13T05:16:51.958616Z","shell.execute_reply":"2022-06-13T05:16:51.977403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nan_values_pct = 100 * round(train_df.isna().sum().compute()/5531451, 4)\nnan_values_pct = nan_values_pct.sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-13T05:22:41.628801Z","iopub.execute_input":"2022-06-13T05:22:41.629195Z","iopub.status.idle":"2022-06-13T05:25:25.126261Z","shell.execute_reply.started":"2022-06-13T05:22:41.629165Z","shell.execute_reply":"2022-06-13T05:25:25.125417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-06-13T05:26:18.670508Z","iopub.execute_input":"2022-06-13T05:26:18.671502Z","iopub.status.idle":"2022-06-13T05:26:18.676018Z","shell.execute_reply.started":"2022-06-13T05:26:18.671457Z","shell.execute_reply":"2022-06-13T05:26:18.67507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nan_values_pct","metadata":{"execution":{"iopub.status.busy":"2022-06-13T05:26:29.21721Z","iopub.execute_input":"2022-06-13T05:26:29.217602Z","iopub.status.idle":"2022-06-13T05:26:29.226861Z","shell.execute_reply.started":"2022-06-13T05:26:29.217573Z","shell.execute_reply":"2022-06-13T05:26:29.225895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot\nfig, ax = plt.subplots()\nax.plot(nan_values_pct)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-13T05:28:46.783994Z","iopub.execute_input":"2022-06-13T05:28:46.784783Z","iopub.status.idle":"2022-06-13T05:28:48.389319Z","shell.execute_reply.started":"2022-06-13T05:28:46.784741Z","shell.execute_reply":"2022-06-13T05:28:48.388404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"del train_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-13T05:30:26.481641Z","iopub.execute_input":"2022-06-13T05:30:26.482121Z","iopub.status.idle":"2022-06-13T05:30:26.944438Z","shell.execute_reply.started":"2022-06-13T05:30:26.482077Z","shell.execute_reply":"2022-06-13T05:30:26.943405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nv1 = train_df['B_31'].value_counts().compute()\nv2 = train_df['D_66'].value_counts().compute()\nv3 = train_df['D_87'].value_counts().compute()\nv4 = train_df['D_114'].value_counts().compute()\nv5 = train_df['D_120'].value_counts().compute()\nv6 = train_df['D_116'].value_counts().compute()","metadata":{"execution":{"iopub.status.busy":"2022-06-10T07:28:41.698411Z","iopub.execute_input":"2022-06-10T07:28:41.699429Z","iopub.status.idle":"2022-06-10T07:36:29.682187Z","shell.execute_reply.started":"2022-06-10T07:28:41.699386Z","shell.execute_reply":"2022-06-10T07:36:29.678296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"| COL   | val =0  | val=1   | COL   | nunique |\n| ----- | ------- | ------- | ----- | ------- |\n| B_31  | 16907   | 5514544 | B_30  | 3       |\n| D_66  | 6288    | 617066  | B_38  | 7       |\n| D_87  |         | 3865    | D_63  | 6       |\n| D_114 | 2038257 | 3316478 | D_64  | 4       |\n| D_116 | 5348109 | 6626    | D_68  | 7       |\n| D_120 | 4729723 | 625012  | D_117 | 7       |\n|       |         |         | D_126 | 3       |\n\n","metadata":{}}]}