{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"# Import libraries\nimport os\nimport warnings\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport datetime as dt\n\nimport gc  # Garbage collector\n\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:14:06.917828Z","iopub.execute_input":"2022-06-23T03:14:06.918356Z","iopub.status.idle":"2022-06-23T03:14:06.945018Z","shell.execute_reply.started":"2022-06-23T03:14:06.918242Z","shell.execute_reply":"2022-06-23T03:14:06.943924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Pre Processing\nReferences\n\nEDA: https://www.kaggle.com/code/awaldeep/first-look-eda/notebook\n    \nAnalysis: https://www.kaggle.com/code/awaldeep/01-starter-xgboost-implementation","metadata":{}},{"cell_type":"code","source":"train_raw = pd.read_feather('../input/amexfeather/train_data.ftr')","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:18:00.964922Z","iopub.execute_input":"2022-06-23T03:18:00.965426Z","iopub.status.idle":"2022-06-23T03:18:25.941681Z","shell.execute_reply.started":"2022-06-23T03:18:00.965389Z","shell.execute_reply":"2022-06-23T03:18:25.939533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_raw.describe()","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:18:28.510013Z","iopub.execute_input":"2022-06-23T03:18:28.511467Z","iopub.status.idle":"2022-06-23T03:21:17.868039Z","shell.execute_reply.started":"2022-06-23T03:18:28.511412Z","shell.execute_reply":"2022-06-23T03:21:17.866656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"missing values","metadata":{}},{"cell_type":"code","source":"tmp = train_raw.isna().sum().mul(100).div(len(train_raw)).sort_values(ascending=False)\nplt.bar(tmp.index[0:50], tmp[0:50])\nplt.show()\nprint(tmp.index[0:20])","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:21:17.871487Z","iopub.execute_input":"2022-06-23T03:21:17.872098Z","iopub.status.idle":"2022-06-23T03:21:24.365952Z","shell.execute_reply.started":"2022-06-23T03:21:17.872046Z","shell.execute_reply":"2022-06-23T03:21:24.364468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"mean data size per a custromer","metadata":{}},{"cell_type":"code","source":"train_raw.shape[0]/train_raw[\"customer_ID\"].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:21:24.368213Z","iopub.execute_input":"2022-06-23T03:21:24.368601Z","iopub.status.idle":"2022-06-23T03:21:25.291798Z","shell.execute_reply.started":"2022-06-23T03:21:24.368568Z","shell.execute_reply":"2022-06-23T03:21:25.289892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Date range\ntrain_raw[\"S_2\"].min(), train_raw[\"S_2\"].max()","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:21:25.295096Z","iopub.execute_input":"2022-06-23T03:21:25.29558Z","iopub.status.idle":"2022-06-23T03:21:25.347995Z","shell.execute_reply.started":"2022-06-23T03:21:25.29554Z","shell.execute_reply":"2022-06-23T03:21:25.346552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Which means below,\n1. 12 datas per a customer\n2. Date range is about 1 year","metadata":{}},{"cell_type":"markdown","source":"range of the each S_2 per a customer","metadata":{}},{"cell_type":"code","source":"train_cus_s2 = train_raw.loc[:, ['customer_ID', 'S_2']].sort_values(['customer_ID', 'S_2'])\ntrain_cus_s2['S_2'].diff()","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:21:25.349861Z","iopub.execute_input":"2022-06-23T03:21:25.350315Z","iopub.status.idle":"2022-06-23T03:21:29.26998Z","shell.execute_reply.started":"2022-06-23T03:21:25.350276Z","shell.execute_reply":"2022-06-23T03:21:29.267995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"diff is not the same range -> check only month diff","metadata":{}},{"cell_type":"code","source":"train_cus_s2[\"S_2_month\"] = train_cus_s2[\"S_2\"].dt.month\ntrain_cus_s2[\"S_2_year\"] = train_cus_s2[\"S_2\"].dt.year","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:21:29.2718Z","iopub.execute_input":"2022-06-23T03:21:29.272238Z","iopub.status.idle":"2022-06-23T03:21:30.372541Z","shell.execute_reply.started":"2022-06-23T03:21:29.272204Z","shell.execute_reply":"2022-06-23T03:21:30.37126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cus_s2[\"S_2_year\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:21:30.37466Z","iopub.execute_input":"2022-06-23T03:21:30.375268Z","iopub.status.idle":"2022-06-23T03:21:30.416702Z","shell.execute_reply.started":"2022-06-23T03:21:30.375211Z","shell.execute_reply":"2022-06-23T03:21:30.415847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cus_s2[\"S_2_year_month\"] = [M+12 \n                                  if Y==2018 \n                                  else M \n                                  for Y, M \n                                  in zip(train_cus_s2[\"S_2_year\"], train_cus_s2[\"S_2_month\"])]","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:21:30.417815Z","iopub.execute_input":"2022-06-23T03:21:30.418414Z","iopub.status.idle":"2022-06-23T03:21:33.097114Z","shell.execute_reply.started":"2022-06-23T03:21:30.418371Z","shell.execute_reply":"2022-06-23T03:21:33.095738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bool_customer_ID = [ID == train_cus_s2['customer_ID'][i] \n                    for i, ID in enumerate(train_cus_s2['customer_ID'][1:])]","metadata":{"execution":{"iopub.status.busy":"2022-06-23T03:21:33.099127Z","iopub.execute_input":"2022-06-23T03:21:33.100141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bool_customer_ID.insert(0, True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cus_s2.loc[bool_customer_ID, 'S_2_diff'].unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_len = len(train_cus_s2['customer_ID'].unique())\nfor i in range(0, 13):\n    diff_len = len(train_cus_s2.query('S_2_diff > @i')['customer_ID'].unique())\n    print(f'\\nMonth_diff:{i}\\n----------\\nnumber:{diff_len}, rate:{diff_len/all_len*100}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some custemor data are not full range in the dates","metadata":{}}]}