{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport cudf\nimport cupy\nimport catboost as cgb\nimport xgboost as lgb\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline\nimport gc\npd.options.display.max_rows=300\npd.options.display.max_columns=300\n\nCATEGORY = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nNUMBER = ['P_2', 'D_39', 'B_1', 'B_2', 'R_1', 'S_3', 'D_41', 'B_3', 'D_42',\n       'D_43', 'D_44', 'B_4', 'D_45', 'B_5', 'R_2', 'D_46', 'D_47',\n       'D_48', 'D_49', 'B_6', 'B_7', 'B_8', 'D_50', 'D_51', 'B_9', 'R_3',\n       'D_52', 'P_3', 'B_10', 'D_53', 'S_5', 'B_11', 'S_6', 'D_54', 'R_4',\n       'S_7', 'B_12', 'S_8', 'D_55', 'D_56', 'B_13', 'R_5', 'D_58', 'S_9',\n       'B_14', 'D_59', 'D_60', 'D_61', 'B_15', 'S_11', 'D_62', 'D_65',\n       'B_16', 'B_17', 'B_18', 'B_19', 'B_20', 'S_12', 'R_6', 'S_13',\n       'B_21', 'D_69', 'B_22', 'D_70', 'D_71', 'D_72', 'S_15', 'B_23',\n       'D_73', 'P_4', 'D_74', 'D_75', 'D_76', 'B_24', 'R_7', 'D_77',\n       'B_25', 'B_26', 'D_78', 'D_79', 'R_8', 'R_9', 'S_16', 'D_80',\n       'R_10', 'R_11', 'B_27', 'D_81', 'D_82', 'S_17', 'R_12', 'B_28',\n       'R_13', 'D_83', 'R_14', 'R_15', 'D_84', 'R_16', 'B_29', 'S_18',\n       'D_86', 'D_87', 'R_17', 'R_18', 'D_88', 'B_31', 'S_19', 'R_19',\n       'B_32', 'S_20', 'R_20', 'R_21', 'B_33', 'D_89', 'R_22', 'R_23',\n       'D_91', 'D_92', 'D_93', 'D_94', 'R_24', 'R_25', 'D_96', 'S_22',\n       'S_23', 'S_24', 'S_25', 'S_26', 'D_102', 'D_103', 'D_104', 'D_105',\n       'D_106', 'D_107', 'B_36', 'B_37', 'R_26', 'R_27', 'D_108', 'D_109',\n       'D_110', 'D_111', 'B_39', 'D_112', 'B_40', 'S_27', 'D_113',\n       'D_115', 'D_118', 'D_119', 'D_121', 'D_122', 'D_123', 'D_124',\n       'D_125', 'D_127', 'D_128', 'D_129', 'B_41', 'B_42', 'D_130',\n       'D_131', 'D_132', 'D_133', 'R_28', 'D_134', 'D_135', 'D_136',\n       'D_137', 'D_138', 'D_139', 'D_140', 'D_141', 'D_142', 'D_143',\n       'D_144', 'D_145']\n\nDATE = ['S_2']","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:22.623698Z","iopub.execute_input":"2022-07-28T14:54:22.624405Z","iopub.status.idle":"2022-07-28T14:54:26.511444Z","shell.execute_reply.started":"2022-07-28T14:54:22.624313Z","shell.execute_reply":"2022-07-28T14:54:26.510503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet(\"../input/boolart-user-default-prediction/train.parquet\")\ntest = pd.read_parquet(\"../input/boolart-user-default-prediction/test.parquet\")\ntargets = pd.read_csv(\"../input/boolart-user-default-prediction/train_labels.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:26.513451Z","iopub.execute_input":"2022-07-28T14:54:26.513901Z","iopub.status.idle":"2022-07-28T14:54:30.136980Z","shell.execute_reply.started":"2022-07-28T14:54:26.513865Z","shell.execute_reply":"2022-07-28T14:54:30.135802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['S_2'] = pd.to_datetime(train['S_2'])\ntest['S_2'] = pd.to_datetime(test['S_2'])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:30.138605Z","iopub.execute_input":"2022-07-28T14:54:30.139039Z","iopub.status.idle":"2022-07-28T14:54:30.246723Z","shell.execute_reply.started":"2022-07-28T14:54:30.139001Z","shell.execute_reply":"2022-07-28T14:54:30.245803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OverView","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:30.249457Z","iopub.execute_input":"2022-07-28T14:54:30.249810Z","iopub.status.idle":"2022-07-28T14:54:30.279019Z","shell.execute_reply.started":"2022-07-28T14:54:30.249775Z","shell.execute_reply":"2022-07-28T14:54:30.269126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:30.283007Z","iopub.execute_input":"2022-07-28T14:54:30.284124Z","iopub.status.idle":"2022-07-28T14:54:30.305822Z","shell.execute_reply.started":"2022-07-28T14:54:30.284067Z","shell.execute_reply":"2022-07-28T14:54:30.304788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:30.307430Z","iopub.execute_input":"2022-07-28T14:54:30.307812Z","iopub.status.idle":"2022-07-28T14:54:30.324200Z","shell.execute_reply.started":"2022-07-28T14:54:30.307775Z","shell.execute_reply":"2022-07-28T14:54:30.323046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Targets","metadata":{}},{"cell_type":"code","source":"targets.target.value_counts().plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:30.326218Z","iopub.execute_input":"2022-07-28T14:54:30.326661Z","iopub.status.idle":"2022-07-28T14:54:30.529481Z","shell.execute_reply.started":"2022-07-28T14:54:30.326619Z","shell.execute_reply":"2022-07-28T14:54:30.528580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_first_date = train.groupby('customer_ID')['S_2'].min()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:30.531028Z","iopub.execute_input":"2022-07-28T14:54:30.531355Z","iopub.status.idle":"2022-07-28T14:54:30.653486Z","shell.execute_reply.started":"2022-07-28T14:54:30.531321Z","shell.execute_reply":"2022-07-28T14:54:30.652518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_first_date.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:30.656016Z","iopub.execute_input":"2022-07-28T14:54:30.656805Z","iopub.status.idle":"2022-07-28T14:54:30.665400Z","shell.execute_reply.started":"2022-07-28T14:54:30.656762Z","shell.execute_reply":"2022-07-28T14:54:30.663958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = targets.merge(customer_first_date.reset_index(), on='customer_ID', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:30.670748Z","iopub.execute_input":"2022-07-28T14:54:30.671125Z","iopub.status.idle":"2022-07-28T14:54:30.711771Z","shell.execute_reply.started":"2022-07-28T14:54:30.671101Z","shell.execute_reply":"2022-07-28T14:54:30.710887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 用户注册时间（首次收到信用卡信息）与违约率。\ntargets.groupby('S_2')['target'].mean().sort_index().plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:30.715090Z","iopub.execute_input":"2022-07-28T14:54:30.715384Z","iopub.status.idle":"2022-07-28T14:54:30.939741Z","shell.execute_reply.started":"2022-07-28T14:54:30.715360Z","shell.execute_reply":"2022-07-28T14:54:30.938779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 时间与用户注册数量\ntargets.groupby('S_2')['target'].count().sort_index().plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:30.941334Z","iopub.execute_input":"2022-07-28T14:54:30.941972Z","iopub.status.idle":"2022-07-28T14:54:31.161803Z","shell.execute_reply.started":"2022-07-28T14:54:30.941933Z","shell.execute_reply":"2022-07-28T14:54:31.160906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Category","metadata":{}},{"cell_type":"code","source":"train[CATEGORY].dtypes","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:31.163128Z","iopub.execute_input":"2022-07-28T14:54:31.164210Z","iopub.status.idle":"2022-07-28T14:54:31.176737Z","shell.execute_reply.started":"2022-07-28T14:54:31.164169Z","shell.execute_reply":"2022-07-28T14:54:31.175792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.merge(targets[['customer_ID', 'target']], on='customer_ID', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:31.178259Z","iopub.execute_input":"2022-07-28T14:54:31.178963Z","iopub.status.idle":"2022-07-28T14:54:37.718691Z","shell.execute_reply.started":"2022-07-28T14:54:31.178925Z","shell.execute_reply":"2022-07-28T14:54:37.717609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_category(col):\n    f, axes = plt.subplots(1, 2, figsize=(16, 2))\n    value_counts = train[col].value_counts()\n    value_counts.plot(kind='bar', xlabel=f'{col} value counts', ax=axes[0])\n    train.groupby(col)['target'].mean().loc[value_counts.index].\\\n    plot(kind='bar', xlabel=f'{col} target mean', ax=axes[1])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:37.720139Z","iopub.execute_input":"2022-07-28T14:54:37.720457Z","iopub.status.idle":"2022-07-28T14:54:37.727394Z","shell.execute_reply.started":"2022-07-28T14:54:37.720423Z","shell.execute_reply":"2022-07-28T14:54:37.726414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in CATEGORY:\n    plot_category(col)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:37.730087Z","iopub.execute_input":"2022-07-28T14:54:37.730582Z","iopub.status.idle":"2022-07-28T14:54:40.554410Z","shell.execute_reply.started":"2022-07-28T14:54:37.730546Z","shell.execute_reply":"2022-07-28T14:54:40.553533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_category_cross_heatmap(col1, col2):\n    cross_mean = train.groupby([col1, col2])['target'].mean()\n    cross_count = train.groupby([col1, col2])['target'].count()\n    \n    f, ax = plt.subplots(1, 2, figsize=(12, 4))\n    sns.heatmap(cross_mean.reset_index().pivot(index=col1, columns=col2, values=\"target\"), \n                annot=True, ax=ax[0])\n    ax[0].set_xlabel(f\"{col1} {col2} cross mean\")\n    \n    sns.heatmap(cross_count.reset_index().pivot(index=col1, columns=col2, values=\"target\"), \n                annot=False, ax=ax[1])\n    ax[1].set_xlabel(f\"{col1} {col2} cross count\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:40.556689Z","iopub.execute_input":"2022-07-28T14:54:40.557415Z","iopub.status.idle":"2022-07-28T14:54:40.566946Z","shell.execute_reply.started":"2022-07-28T14:54:40.557373Z","shell.execute_reply":"2022-07-28T14:54:40.565878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor col1 in CATEGORY:\n    for col2 in CATEGORY:\n        if col2 != col1:\n            plot_category_cross_heatmap(col1, col2)\n        ","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:54:40.569159Z","iopub.execute_input":"2022-07-28T14:54:40.569898Z","iopub.status.idle":"2022-07-28T14:55:45.910968Z","shell.execute_reply.started":"2022-07-28T14:54:40.569831Z","shell.execute_reply":"2022-07-28T14:55:45.909904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cross heatmap 中，任何不寻常的交叉区域，都可以成为特征，但还是要考虑frequency，频次太低代表性不足。\n# 例如 D_68 与 D_63 图中的 0.029/0.038格，基本可以断定是不可能违约用户。\n# 例如 D_30 与 D_11 图中的交叉点\n# 特征数量非常多时，数模型可能比较难以发现有效的特征组合。","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:45.912304Z","iopub.execute_input":"2022-07-28T14:55:45.914112Z","iopub.status.idle":"2022-07-28T14:55:45.918412Z","shell.execute_reply.started":"2022-07-28T14:55:45.914072Z","shell.execute_reply":"2022-07-28T14:55:45.917401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check category distribution difference between train and test \n\nfor col in CATEGORY:\n    f, ax = plt.subplots(1, 2, figsize=(12,4))\n    ax[0].set_title(f\"{col} train\")\n    trn_cnt = train[col].value_counts()\n    trn_cnt.plot(kind='bar', ax=ax[0])\n    \n    ax[1].set_title(f\"{col} test\")\n    test[col].value_counts().plot(kind='bar', ax=ax[1])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:45.920019Z","iopub.execute_input":"2022-07-28T14:55:45.920631Z","iopub.status.idle":"2022-07-28T14:55:49.006831Z","shell.execute_reply.started":"2022-07-28T14:55:45.920594Z","shell.execute_reply":"2022-07-28T14:55:49.005749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 如果不考虑缺失值，D_66 属于无效特征，test dataset 不存在 0 类别。\n# 总体来看train/test 分布差异比较小，但D_120分布差异较大","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:49.008414Z","iopub.execute_input":"2022-07-28T14:55:49.008883Z","iopub.status.idle":"2022-07-28T14:55:49.014059Z","shell.execute_reply.started":"2022-07-28T14:55:49.008829Z","shell.execute_reply":"2022-07-28T14:55:49.012905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# D_120只有2个取值，可以计算 value_mean 在时间上的变化\n\ntrain.groupby('S_2')['D_120'].mean().sort_index().plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:49.015787Z","iopub.execute_input":"2022-07-28T14:55:49.016168Z","iopub.status.idle":"2022-07-28T14:55:49.371858Z","shell.execute_reply.started":"2022-07-28T14:55:49.016131Z","shell.execute_reply":"2022-07-28T14:55:49.370966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby('S_2')['D_120'].mean().sort_index().plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:49.373353Z","iopub.execute_input":"2022-07-28T14:55:49.373708Z","iopub.status.idle":"2022-07-28T14:55:49.716688Z","shell.execute_reply.started":"2022-07-28T14:55:49.373673Z","shell.execute_reply":"2022-07-28T14:55:49.715763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 随着时间变化，120 有增长趋势。","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:49.718277Z","iopub.execute_input":"2022-07-28T14:55:49.718937Z","iopub.status.idle":"2022-07-28T14:55:49.722979Z","shell.execute_reply.started":"2022-07-28T14:55:49.718901Z","shell.execute_reply":"2022-07-28T14:55:49.721995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# category  缺失数据分析\n\ntrain[CATEGORY].isnull().mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:49.724589Z","iopub.execute_input":"2022-07-28T14:55:49.725293Z","iopub.status.idle":"2022-07-28T14:55:49.887712Z","shell.execute_reply.started":"2022-07-28T14:55:49.725258Z","shell.execute_reply":"2022-07-28T14:55:49.886672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[CATEGORY].isnull().mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:49.889246Z","iopub.execute_input":"2022-07-28T14:55:49.889805Z","iopub.status.idle":"2022-07-28T14:55:49.903443Z","shell.execute_reply.started":"2022-07-28T14:55:49.889761Z","shell.execute_reply":"2022-07-28T14:55:49.902199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 大部分数据缺失率很低，除了D_66, train/test 缺失率基本保持一致","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:49.904893Z","iopub.execute_input":"2022-07-28T14:55:49.905569Z","iopub.status.idle":"2022-07-28T14:55:49.910342Z","shell.execute_reply.started":"2022-07-28T14:55:49.905534Z","shell.execute_reply":"2022-07-28T14:55:49.909125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 每个用户，在1-13个月信用卡账单中，每个category feature 出现的个数\n\nfor col in CATEGORY:\n    f, ax = plt.subplots(1, 2, figsize=(12, 4))\n    train.groupby('customer_ID')[col].nunique().agg(['min', 'max', 'mean']).plot(kind='bar', ax=ax[0])\n    test.groupby('customer_ID')[col].nunique().agg(['min', 'max', 'mean']).plot(kind='bar', ax=ax[1])\n    \n    ax[0].set_title(f\"{col} count desc in train\")\n    ax[1].set_title(f\"{col} count desc in test\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:49.922390Z","iopub.execute_input":"2022-07-28T14:55:49.924391Z","iopub.status.idle":"2022-07-28T14:55:54.318822Z","shell.execute_reply.started":"2022-07-28T14:55:49.924344Z","shell.execute_reply":"2022-07-28T14:55:54.317855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# min=0 意味着用户数据缺失\n# max/mean 较大意味着该变量在13个月账单中有比较大的变化。","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:54.320101Z","iopub.execute_input":"2022-07-28T14:55:54.320963Z","iopub.status.idle":"2022-07-28T14:55:54.325309Z","shell.execute_reply.started":"2022-07-28T14:55:54.320922Z","shell.execute_reply":"2022-07-28T14:55:54.324383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(CATEGORY, axis=1, inplace=True)\ntest.drop(CATEGORY, axis=1, inplace=True)\n\nimport gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:54.326683Z","iopub.execute_input":"2022-07-28T14:55:54.327607Z","iopub.status.idle":"2022-07-28T14:55:56.038172Z","shell.execute_reply.started":"2022-07-28T14:55:54.327572Z","shell.execute_reply":"2022-07-28T14:55:56.037055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Numerical","metadata":{}},{"cell_type":"code","source":"# 根据比赛数据描述，不同前缀代表含义\n# D_* = Delinquency variables\n# S_* = Spend variables\n# P_* = Payment variables\n# B_* = Balance variables\n# R_* = Risk variables","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:56.039560Z","iopub.execute_input":"2022-07-28T14:55:56.040006Z","iopub.status.idle":"2022-07-28T14:55:56.044728Z","shell.execute_reply.started":"2022-07-28T14:55:56.039969Z","shell.execute_reply":"2022-07-28T14:55:56.043720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(NUMBER)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:56.046265Z","iopub.execute_input":"2022-07-28T14:55:56.046648Z","iopub.status.idle":"2022-07-28T14:55:56.059604Z","shell.execute_reply.started":"2022-07-28T14:55:56.046569Z","shell.execute_reply":"2022-07-28T14:55:56.058686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMBER[:2]","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:56.062424Z","iopub.execute_input":"2022-07-28T14:55:56.062675Z","iopub.status.idle":"2022-07-28T14:55:56.070849Z","shell.execute_reply.started":"2022-07-28T14:55:56.062647Z","shell.execute_reply":"2022-07-28T14:55:56.069855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 177个特征，先做一些基本统计 train/test  min/mean/max/std/isnull_mean/nunique\n# 因为已经得知，numerical feature 添加了 uniform(0.00, 0.01) 噪音\n# ref: https://www.kaggle.com/competitions/amex-default-prediction/discussion/327649\n# 因此我们再统计一下 round2/3/6 nunique\n\n# 为了加速计算，采用cudl","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:56.072460Z","iopub.execute_input":"2022-07-28T14:55:56.072889Z","iopub.status.idle":"2022-07-28T14:55:56.081482Z","shell.execute_reply.started":"2022-07-28T14:55:56.072855Z","shell.execute_reply":"2022-07-28T14:55:56.080410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def numerical_desc(df, col):\n    cu_series = cudf.Series(df[col])\n    desc = []\n    \n    desc.append(cu_series.min())\n    desc.append(cu_series.mean())\n    desc.append(cu_series.max())\n    desc.append(cu_series.std())\n    desc.append(cu_series.null_count / len(cu_series))\n    desc.append(cu_series.nunique())\n    \n    desc.append(cu_series.round(2).nunique())\n    desc.append(cu_series.round(3).nunique())\n    desc.append(cu_series.round(6).nunique())\n    \n    del cu_series; gc.collect()\n    return desc","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:56.082574Z","iopub.execute_input":"2022-07-28T14:55:56.084453Z","iopub.status.idle":"2022-07-28T14:55:56.092914Z","shell.execute_reply.started":"2022-07-28T14:55:56.084407Z","shell.execute_reply":"2022-07-28T14:55:56.091887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train desc \ntrain_desc = {}\nfor col in NUMBER:\n    train_desc[col] = numerical_desc(train, col)\ntrain_desc = pd.DataFrame(train_desc).T\ntrain_desc.columns = ['min', 'mean', 'max', 'std', 'null', 'nunique', 'r2_nunique', 'r3_nunique', 'r6_nunique']","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:55:56.095799Z","iopub.execute_input":"2022-07-28T14:55:56.096066Z","iopub.status.idle":"2022-07-28T14:56:21.913719Z","shell.execute_reply.started":"2022-07-28T14:55:56.096044Z","shell.execute_reply":"2022-07-28T14:56:21.912800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test desc \ntest_desc = {}\nfor col in NUMBER:\n    test_desc[col] = numerical_desc(test, col)\ntest_desc = pd.DataFrame(test_desc).T\ntest_desc.columns = ['min', 'mean', 'max', 'std', 'null', 'nunique', 'r2_nunique', 'r3_nunique', 'r6_nunique']","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:21.914975Z","iopub.execute_input":"2022-07-28T14:56:21.916659Z","iopub.status.idle":"2022-07-28T14:56:44.509110Z","shell.execute_reply.started":"2022-07-28T14:56:21.916622Z","shell.execute_reply":"2022-07-28T14:56:44.508121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train_desc.columns:\n    plt.figure(figsize=(12, 4))\n    plt.plot(train_desc[col].values, label='train')\n    plt.plot(test_desc[col].values, label='test')\n    plt.title(col)\n    plt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:44.510554Z","iopub.execute_input":"2022-07-28T14:56:44.510929Z","iopub.status.idle":"2022-07-28T14:56:46.283704Z","shell.execute_reply.started":"2022-07-28T14:56:44.510895Z","shell.execute_reply":"2022-07-28T14:56:46.282764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc['min'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.285300Z","iopub.execute_input":"2022-07-28T14:56:46.285664Z","iopub.status.idle":"2022-07-28T14:56:46.299183Z","shell.execute_reply.started":"2022-07-28T14:56:46.285629Z","shell.execute_reply":"2022-07-28T14:56:46.297996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 大部分数值变量的最小值都在0附近 25~75分位数 < 1e-8","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.300891Z","iopub.execute_input":"2022-07-28T14:56:46.301221Z","iopub.status.idle":"2022-07-28T14:56:46.305889Z","shell.execute_reply.started":"2022-07-28T14:56:46.301188Z","shell.execute_reply":"2022-07-28T14:56:46.304894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc['max'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.307191Z","iopub.execute_input":"2022-07-28T14:56:46.308215Z","iopub.status.idle":"2022-07-28T14:56:46.320645Z","shell.execute_reply.started":"2022-07-28T14:56:46.308179Z","shell.execute_reply":"2022-07-28T14:56:46.319674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 一部分变量最大值在 1 附近， 大部分变量最大值 < 15","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.322023Z","iopub.execute_input":"2022-07-28T14:56:46.322355Z","iopub.status.idle":"2022-07-28T14:56:46.329945Z","shell.execute_reply.started":"2022-07-28T14:56:46.322320Z","shell.execute_reply":"2022-07-28T14:56:46.328611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc['mean'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.331932Z","iopub.execute_input":"2022-07-28T14:56:46.332701Z","iopub.status.idle":"2022-07-28T14:56:46.345348Z","shell.execute_reply.started":"2022-07-28T14:56:46.332663Z","shell.execute_reply":"2022-07-28T14:56:46.344385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 所有变量的均值都在0-1区间","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.348242Z","iopub.execute_input":"2022-07-28T14:56:46.349206Z","iopub.status.idle":"2022-07-28T14:56:46.355436Z","shell.execute_reply.started":"2022-07-28T14:56:46.349146Z","shell.execute_reply":"2022-07-28T14:56:46.354589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc['null'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.356724Z","iopub.execute_input":"2022-07-28T14:56:46.359073Z","iopub.status.idle":"2022-07-28T14:56:46.370645Z","shell.execute_reply.started":"2022-07-28T14:56:46.359033Z","shell.execute_reply":"2022-07-28T14:56:46.369652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc['null'].sort_values(ascending=False).head(5)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.373640Z","iopub.execute_input":"2022-07-28T14:56:46.374275Z","iopub.status.idle":"2022-07-28T14:56:46.382963Z","shell.execute_reply.started":"2022-07-28T14:56:46.374240Z","shell.execute_reply":"2022-07-28T14:56:46.381993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc.loc['D_87']","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.384032Z","iopub.execute_input":"2022-07-28T14:56:46.387216Z","iopub.status.idle":"2022-07-28T14:56:46.395964Z","shell.execute_reply.started":"2022-07-28T14:56:46.387179Z","shell.execute_reply":"2022-07-28T14:56:46.394897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 大部分变量缺失很少，少部分变量有99%以上的缺失数值","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.397722Z","iopub.execute_input":"2022-07-28T14:56:46.398519Z","iopub.status.idle":"2022-07-28T14:56:46.405761Z","shell.execute_reply.started":"2022-07-28T14:56:46.398447Z","shell.execute_reply":"2022-07-28T14:56:46.404755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby(train['D_87'].fillna(-1))['target'].mean()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.407254Z","iopub.execute_input":"2022-07-28T14:56:46.407936Z","iopub.status.idle":"2022-07-28T14:56:46.435678Z","shell.execute_reply.started":"2022-07-28T14:56:46.407902Z","shell.execute_reply":"2022-07-28T14:56:46.434844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Delinquency 是关于拖欠相关特征，D_87 缺失 -> 没有拖欠征信记录？ -> 违约率下降 ？","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.437113Z","iopub.execute_input":"2022-07-28T14:56:46.437423Z","iopub.status.idle":"2022-07-28T14:56:46.441778Z","shell.execute_reply.started":"2022-07-28T14:56:46.437392Z","shell.execute_reply":"2022-07-28T14:56:46.440875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc['nunique'].plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:46.442998Z","iopub.execute_input":"2022-07-28T14:56:46.443565Z","iopub.status.idle":"2022-07-28T14:56:48.075771Z","shell.execute_reply.started":"2022-07-28T14:56:46.443530Z","shell.execute_reply":"2022-07-28T14:56:48.074796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 由于添加了噪音，绝大部分特征 nunique 完全相等","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.077145Z","iopub.execute_input":"2022-07-28T14:56:48.077637Z","iopub.status.idle":"2022-07-28T14:56:48.084420Z","shell.execute_reply.started":"2022-07-28T14:56:48.077595Z","shell.execute_reply":"2022-07-28T14:56:48.083498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc[train_desc['nunique'] < 1e5]","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.085821Z","iopub.execute_input":"2022-07-28T14:56:48.086270Z","iopub.status.idle":"2022-07-28T14:56:48.180832Z","shell.execute_reply.started":"2022-07-28T14:56:48.086234Z","shell.execute_reply":"2022-07-28T14:56:48.179065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# nunique 小的要么只有1个变量，要么缺失值很多","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.185499Z","iopub.execute_input":"2022-07-28T14:56:48.185914Z","iopub.status.idle":"2022-07-28T14:56:48.190917Z","shell.execute_reply.started":"2022-07-28T14:56:48.185844Z","shell.execute_reply":"2022-07-28T14:56:48.189898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_desc['r6_nunique'] / train_desc['nunique']).plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.192829Z","iopub.execute_input":"2022-07-28T14:56:48.195122Z","iopub.status.idle":"2022-07-28T14:56:48.453929Z","shell.execute_reply.started":"2022-07-28T14:56:48.195078Z","shell.execute_reply":"2022-07-28T14:56:48.452894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc['r6_nunique'].max() / train_desc['nunique'].max()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.457241Z","iopub.execute_input":"2022-07-28T14:56:48.460392Z","iopub.status.idle":"2022-07-28T14:56:48.471556Z","shell.execute_reply.started":"2022-07-28T14:56:48.460350Z","shell.execute_reply":"2022-07-28T14:56:48.470472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# round 6 削减了80% nunique","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.473932Z","iopub.execute_input":"2022-07-28T14:56:48.474500Z","iopub.status.idle":"2022-07-28T14:56:48.479412Z","shell.execute_reply.started":"2022-07-28T14:56:48.474456Z","shell.execute_reply":"2022-07-28T14:56:48.478082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc['r3_nunique'].max() / train_desc['nunique'].max()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.481714Z","iopub.execute_input":"2022-07-28T14:56:48.482530Z","iopub.status.idle":"2022-07-28T14:56:48.497439Z","shell.execute_reply.started":"2022-07-28T14:56:48.482492Z","shell.execute_reply":"2022-07-28T14:56:48.496487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train_desc['r3_nunique'] / train_desc['nunique']).plot()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.499705Z","iopub.execute_input":"2022-07-28T14:56:48.500397Z","iopub.status.idle":"2022-07-28T14:56:48.702659Z","shell.execute_reply.started":"2022-07-28T14:56:48.500362Z","shell.execute_reply":"2022-07-28T14:56:48.701527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# round 3 削减了99.5% nunique","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.704457Z","iopub.execute_input":"2022-07-28T14:56:48.705165Z","iopub.status.idle":"2022-07-28T14:56:48.714199Z","shell.execute_reply.started":"2022-07-28T14:56:48.705125Z","shell.execute_reply":"2022-07-28T14:56:48.713160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_desc['r2_nunique'].max() / train_desc['nunique'].max()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.715832Z","iopub.execute_input":"2022-07-28T14:56:48.716812Z","iopub.status.idle":"2022-07-28T14:56:48.727098Z","shell.execute_reply.started":"2022-07-28T14:56:48.716759Z","shell.execute_reply":"2022-07-28T14:56:48.726064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# round 2 削减了99.9% nunique","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.729017Z","iopub.execute_input":"2022-07-28T14:56:48.729906Z","iopub.status.idle":"2022-07-28T14:56:48.736887Z","shell.execute_reply.started":"2022-07-28T14:56:48.729844Z","shell.execute_reply":"2022-07-28T14:56:48.735865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 选取特征P_2 和并train/test 一起观察\n\np2_train = cudf.DataFrame(train[['customer_ID', 'S_2', 'P_2', 'target']])\np2_test = cudf.DataFrame(test[['customer_ID', 'S_2', 'P_2']])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.738818Z","iopub.execute_input":"2022-07-28T14:56:48.739569Z","iopub.status.idle":"2022-07-28T14:56:48.908536Z","shell.execute_reply.started":"2022-07-28T14:56:48.739531Z","shell.execute_reply":"2022-07-28T14:56:48.907648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p2 = cudf.concat([p2_train, p2_test], axis=0, ignore_index=True)\ndel p2_train; del p2_test; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:48.912481Z","iopub.execute_input":"2022-07-28T14:56:48.913046Z","iopub.status.idle":"2022-07-28T14:56:49.121387Z","shell.execute_reply.started":"2022-07-28T14:56:48.913012Z","shell.execute_reply":"2022-07-28T14:56:49.120490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p2.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:49.122897Z","iopub.execute_input":"2022-07-28T14:56:49.123243Z","iopub.status.idle":"2022-07-28T14:56:49.129730Z","shell.execute_reply.started":"2022-07-28T14:56:49.123208Z","shell.execute_reply":"2022-07-28T14:56:49.128698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p2['P_2'].nunique()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:49.131436Z","iopub.execute_input":"2022-07-28T14:56:49.132154Z","iopub.status.idle":"2022-07-28T14:56:49.144211Z","shell.execute_reply.started":"2022-07-28T14:56:49.132118Z","shell.execute_reply":"2022-07-28T14:56:49.143140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1689万条数据几乎没有重复值。","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:49.145961Z","iopub.execute_input":"2022-07-28T14:56:49.146377Z","iopub.status.idle":"2022-07-28T14:56:49.151039Z","shell.execute_reply.started":"2022-07-28T14:56:49.146342Z","shell.execute_reply":"2022-07-28T14:56:49.149996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p2['r2'] = p2['P_2'].round(2)\np2['r3'] = p2['P_2'].round(3)\np2['r6'] = p2['P_2'].round(6)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:49.152698Z","iopub.execute_input":"2022-07-28T14:56:49.153487Z","iopub.status.idle":"2022-07-28T14:56:49.162225Z","shell.execute_reply.started":"2022-07-28T14:56:49.153450Z","shell.execute_reply":"2022-07-28T14:56:49.161335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for rlvl in ['r2', 'r3', 'r6']:\n    print(f\"round level {rlvl} nunique {p2[rlvl].nunique()}\")\nprint(f\"orignal nunique {p2['P_2'].nunique()}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:49.164729Z","iopub.execute_input":"2022-07-28T14:56:49.165665Z","iopub.status.idle":"2022-07-28T14:56:49.183820Z","shell.execute_reply.started":"2022-07-28T14:56:49.165628Z","shell.execute_reply":"2022-07-28T14:56:49.182686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 噪音 0~0.01\n# 每个customer 最后时间的取值观察(数据已经按照时间大小排序，取每个用户最后一行即可)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:49.185649Z","iopub.execute_input":"2022-07-28T14:56:49.185987Z","iopub.status.idle":"2022-07-28T14:56:49.190648Z","shell.execute_reply.started":"2022-07-28T14:56:49.185955Z","shell.execute_reply":"2022-07-28T14:56:49.189606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p2_lastdate = p2.groupby(\"customer_ID\").last().reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:49.192098Z","iopub.execute_input":"2022-07-28T14:56:49.192539Z","iopub.status.idle":"2022-07-28T14:56:49.383225Z","shell.execute_reply.started":"2022-07-28T14:56:49.192505Z","shell.execute_reply":"2022-07-28T14:56:49.379048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for rlvl in ['r2', 'r3', 'r6']:\n    print(f\"round level {rlvl} nunique {p2_lastdate[rlvl].nunique()}\")\nprint(f\"orignal nunique {p2_lastdate['P_2'].nunique()}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:49.384676Z","iopub.execute_input":"2022-07-28T14:56:49.389058Z","iopub.status.idle":"2022-07-28T14:56:49.398920Z","shell.execute_reply.started":"2022-07-28T14:56:49.389031Z","shell.execute_reply":"2022-07-28T14:56:49.397083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for r_level in ['r2', 'r3', 'r6', 'P_2']:\n    plt.figure(figsize=(12, 4))\n    p2_lastdate.groupby(r_level)['target'].mean().sort_index().to_pandas().plot()\n    corr = p2_lastdate[[r_level, 'target']].to_pandas().corr().iloc[0, 1]\n    plt.title(f\"P_2 round level {r_level} corr {corr:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:49.400801Z","iopub.execute_input":"2022-07-28T14:56:49.401378Z","iopub.status.idle":"2022-07-28T14:56:50.351010Z","shell.execute_reply.started":"2022-07-28T14:56:49.401343Z","shell.execute_reply":"2022-07-28T14:56:50.350007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# round2/3 具有比较强的统计性（每个取值包含了很多样本）","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:50.352345Z","iopub.execute_input":"2022-07-28T14:56:50.353392Z","iopub.status.idle":"2022-07-28T14:56:50.358213Z","shell.execute_reply.started":"2022-07-28T14:56:50.353351Z","shell.execute_reply":"2022-07-28T14:56:50.357052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 尝试对频率进行编码\nf, ax = plt.subplots(1, 6, figsize=(20, 4))\nf.suptitle(\"P2\", fontsize=16)\nfor i, r_lvl in enumerate(['r2', 'r3', 'r6']):\n    \n    p2_lastdate.groupby([r_lvl]).target.agg(['count', 'mean'])\\\n    .sort_values(\"count\").set_index(\"count\").to_pandas().plot(ax=ax[0+i*2])\n    ax[0+i*2].set_xlabel(f\"freq & target mean\")\n    \n    p2_lastdate.groupby([r_lvl]).target.agg(['count']).reset_index()\\\n    .sort_values(\"count\").set_index(\"count\").to_pandas()[r_lvl].plot(ax=ax[1+i*2])\n    ax[1+i*2].set_xlabel(f\"freq & value\")\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:50.360853Z","iopub.execute_input":"2022-07-28T14:56:50.361437Z","iopub.status.idle":"2022-07-28T14:56:51.268880Z","shell.execute_reply.started":"2022-07-28T14:56:50.361409Z","shell.execute_reply":"2022-07-28T14:56:51.267795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(p2_lastdate.groupby('r2')['target'].agg(['count', 'mean']).reset_index().to_pandas().corr())","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:51.270623Z","iopub.execute_input":"2022-07-28T14:56:51.271087Z","iopub.status.idle":"2022-07-28T14:56:51.290371Z","shell.execute_reply.started":"2022-07-28T14:56:51.271039Z","shell.execute_reply":"2022-07-28T14:56:51.289494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(p2_lastdate.groupby('r3')['target'].agg(['count', 'mean']).reset_index().to_pandas().corr())","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:51.291597Z","iopub.execute_input":"2022-07-28T14:56:51.292025Z","iopub.status.idle":"2022-07-28T14:56:51.310369Z","shell.execute_reply.started":"2022-07-28T14:56:51.291990Z","shell.execute_reply":"2022-07-28T14:56:51.309064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(p2_lastdate.groupby('r6')['target'].agg(['count', 'mean']).reset_index().to_pandas().corr())","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:51.312862Z","iopub.execute_input":"2022-07-28T14:56:51.313864Z","iopub.status.idle":"2022-07-28T14:56:51.334764Z","shell.execute_reply.started":"2022-07-28T14:56:51.313815Z","shell.execute_reply":"2022-07-28T14:56:51.333898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pair = p2_lastdate.groupby('r6')['target'].agg(['count', 'mean']).reset_index().to_pandas()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:51.335782Z","iopub.execute_input":"2022-07-28T14:56:51.336066Z","iopub.status.idle":"2022-07-28T14:56:51.351865Z","shell.execute_reply.started":"2022-07-28T14:56:51.336042Z","shell.execute_reply":"2022-07-28T14:56:51.351011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# r2/r3 频率编码显示良好的统计性能...\n# 尝试对其他连续型特征编码验证\n\ndef freqency_encode_analysis(col):\n    df = cudf.DataFrame(train[[col, 'customer_ID', 'target']])\n    df_lastdate = df.groupby(\"customer_ID\").last().reset_index()\n    \n    df_lastdate['r2'] = df_lastdate[col].round(2)\n    df_lastdate['r3'] = df_lastdate[col].round(3)\n    df_lastdate['r6'] = df_lastdate[col].round(6)\n    \n    f, ax = plt.subplots(2, 6, figsize=(24, 10))\n    \n    for i, r_lvl in enumerate(['r2', 'r3', 'r6']):\n        df_lastdate.groupby(r_lvl)['target'].mean().sort_index().to_pandas().plot(ax=ax[1][i])\n        corr = df_lastdate[[r_lvl, 'target']].to_pandas().corr().iloc[0, 1]\n        ax[1][i].set_title(f\"{r_lvl} corr {corr:.4f}\")\n        ax[1][i].set_xlabel(f\"value\")\n        ax[1][i].set_ylabel(\"target mean\")\n        \n        df_lastdate.groupby([r_lvl]).target.agg(['count', 'mean'])\\\n        .sort_values(\"count\").set_index(\"count\").to_pandas().plot(ax=ax[0][0+i*2])\n        ax[0][0+i*2].set_title(f\"{r_lvl} freq & target mean\")\n\n        df_lastdate.groupby([r_lvl]).target.agg(['count']).reset_index()\\\n        .sort_values(\"count\").set_index(\"count\").to_pandas()[r_lvl].plot(ax=ax[0][1+i*2])\n        ax[0][1+i*2].set_title(f\"{r_lvl} freq & value\")\n        \n        pair = df_lastdate.groupby([r_lvl]).target.agg(['count', 'mean']).reset_index()\n        \n        \n        \n    r2_corr = df_lastdate.groupby('r2')['target'].agg(['count', 'mean']).reset_index().to_pandas().corr()\n    r6_corr = df_lastdate.groupby('r6')['target'].agg(['count', 'mean']).reset_index().to_pandas().corr()\n    \n    r2_corr_info = f\"r2 corr: value-freq  {r2_corr['r2']['count']:.3f} value-target {r2_corr['r2']['mean']:.3f} freq-target {r2_corr['count']['mean']}\"\n    r6_corr_info = f\"r6 corr: value-freq  {r6_corr['r6']['count']:.3f} value-target {r6_corr['r6']['mean']:.3f} freq-target {r6_corr['count']['mean']}\"\n    \n    f.suptitle(f\"{col}  \\n\" + r2_corr_info + \"\\n\" + r6_corr_info, fontsize=16)\n    plt.tight_layout()\n    \n    del df; del df_lastdate; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:51.354899Z","iopub.execute_input":"2022-07-28T14:56:51.355176Z","iopub.status.idle":"2022-07-28T14:56:51.370077Z","shell.execute_reply.started":"2022-07-28T14:56:51.355154Z","shell.execute_reply":"2022-07-28T14:56:51.368950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in NUMBER:\n\n    freqency_encode_analysis(col)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:56:51.371323Z","iopub.execute_input":"2022-07-28T14:56:51.372269Z","iopub.status.idle":"2022-07-28T15:06:31.863404Z","shell.execute_reply.started":"2022-07-28T14:56:51.372232Z","shell.execute_reply":"2022-07-28T15:06:31.862152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Joint Distribution","metadata":{}},{"cell_type":"code","source":"train_last = train.groupby(\"customer_ID\").agg(\"last\").reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:34.780293Z","iopub.execute_input":"2022-07-28T15:12:34.781008Z","iopub.status.idle":"2022-07-28T15:12:35.782566Z","shell.execute_reply.started":"2022-07-28T15:12:34.780970Z","shell.execute_reply":"2022-07-28T15:12:35.781609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"P2 = pd.qcut(train_last['P_2'], 5, labels=[f\"{col}_{i}\" for i in range(5)])\n\ntrain_last.groupby(P2).target.mean().plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:35.806255Z","iopub.execute_input":"2022-07-28T15:12:35.806541Z","iopub.status.idle":"2022-07-28T15:12:35.985658Z","shell.execute_reply.started":"2022-07-28T15:12:35.806516Z","shell.execute_reply":"2022-07-28T15:12:35.984698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"B9 = pd.qcut(train_last['B_9'], 5, labels=[f\"{col}_{i}\" for i in range(5)])\n\ntrain_last.groupby(B9).target.mean().plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:37.857773Z","iopub.execute_input":"2022-07-28T15:12:37.858452Z","iopub.status.idle":"2022-07-28T15:12:38.035832Z","shell.execute_reply.started":"2022-07-28T15:12:37.858417Z","shell.execute_reply":"2022-07-28T15:12:38.034976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_last['B9_code'] = B9\ntrain_last['P2_code'] = P2\n\njoint_mean = train_last.groupby(['B9_code', 'P2_code']).target.mean().reset_index()\\\n.pivot(index=\"B9_code\", columns='P2_code', values='target')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:39.159639Z","iopub.execute_input":"2022-07-28T15:12:39.160017Z","iopub.status.idle":"2022-07-28T15:12:39.176730Z","shell.execute_reply.started":"2022-07-28T15:12:39.159968Z","shell.execute_reply":"2022-07-28T15:12:39.175694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8, 6))\nsns.heatmap(joint_mean, annot=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:40.103313Z","iopub.execute_input":"2022-07-28T15:12:40.103660Z","iopub.status.idle":"2022-07-28T15:12:40.503541Z","shell.execute_reply.started":"2022-07-28T15:12:40.103630Z","shell.execute_reply":"2022-07-28T15:12:40.502648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Field feature statistic","metadata":{}},{"cell_type":"code","source":"# D_* = Delinquency variables\n# S_* = Spend variables\n# P_* = Payment variables\n# B_* = Balance variables\n# R_* = Risk variables","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:41.870732Z","iopub.execute_input":"2022-07-28T15:12:41.871585Z","iopub.status.idle":"2022-07-28T15:12:41.876680Z","shell.execute_reply.started":"2022-07-28T15:12:41.871539Z","shell.execute_reply":"2022-07-28T15:12:41.875876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"D_nums = [col for col in NUMBER if col.startswith('D')]","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:42.407395Z","iopub.execute_input":"2022-07-28T15:12:42.407732Z","iopub.status.idle":"2022-07-28T15:12:42.412901Z","shell.execute_reply.started":"2022-07-28T15:12:42.407702Z","shell.execute_reply":"2022-07-28T15:12:42.411901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def field_analysis(field, num_round=1):\n    field_nums = [col for col in NUMBER if col.startswith(field)]\n    # D_num sum round 2\n    f, ax = plt.subplots(1,3,figsize=(12, 4))\n    plt.suptitle(f\"field {field} round {num_round}\")\n    tmp = train_last.groupby(train_last[field_nums]\\\n                             .mean(axis=1).round(num_round)).target.mean().reset_index()\n    \n    sns.regplot(x=\"index\", y=\"target\", data=tmp, ax=ax[0])\n    sns.kdeplot(x=\"index\", y=\"target\", data=tmp, ax=ax[0])\n    ax[0].set_title(f\"mean vs. target\")\n    \n    tmp = train.groupby(train_last[field_nums]\\\n                  .std(axis=1).round(num_round)).target.std().reset_index()\n    sns.regplot(x=\"index\", y=\"target\", data=tmp, ax=ax[1])\n    sns.kdeplot(x=\"index\", y=\"target\", data=tmp, ax=ax[1])\n    ax[1].set_title(f\"std vs. target\")\n    \n    tmp = train_last.groupby(train_last[field_nums].isnull().sum(axis=1)).target.mean().reset_index()\n\n    # f, ax = plt.subplots(figsize=(6, 6))\n    sns.regplot(x=\"index\", y=\"target\", data=tmp, ax=ax[2])\n    sns.kdeplot(x=\"index\", y=\"target\", data=tmp, ax=ax[2])\n    ax[2].set_title(f\"null sum vs. target\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:42.999597Z","iopub.execute_input":"2022-07-28T15:12:43.000667Z","iopub.status.idle":"2022-07-28T15:12:43.012223Z","shell.execute_reply.started":"2022-07-28T15:12:43.000616Z","shell.execute_reply":"2022-07-28T15:12:43.011186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# D_* = Delinquency variables Summary\n# D_* 拖欠特征均值越高，违约率越高， mean 较为显著， std 不显著 ，缺失数据越多违约率越低\n\nfield_analysis(\"D\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:44.151848Z","iopub.execute_input":"2022-07-28T15:12:44.152928Z","iopub.status.idle":"2022-07-28T15:12:45.275734Z","shell.execute_reply.started":"2022-07-28T15:12:44.152874Z","shell.execute_reply":"2022-07-28T15:12:45.274159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# P_* = Payment variables\n\nfield_analysis(\"P\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:45.278998Z","iopub.execute_input":"2022-07-28T15:12:45.279417Z","iopub.status.idle":"2022-07-28T15:12:46.331189Z","shell.execute_reply.started":"2022-07-28T15:12:45.279374Z","shell.execute_reply":"2022-07-28T15:12:46.329822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# S_* = Spend variables\nfield_analysis(\"S\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:46.333101Z","iopub.execute_input":"2022-07-28T15:12:46.333547Z","iopub.status.idle":"2022-07-28T15:12:47.916741Z","shell.execute_reply.started":"2022-07-28T15:12:46.333509Z","shell.execute_reply":"2022-07-28T15:12:47.915801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# B_* = Balance variables\nfield_analysis(\"B\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:12:47.919151Z","iopub.execute_input":"2022-07-28T15:12:47.920107Z","iopub.status.idle":"2022-07-28T15:12:48.893631Z","shell.execute_reply.started":"2022-07-28T15:12:47.920068Z","shell.execute_reply":"2022-07-28T15:12:48.892733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feild Cross","metadata":{}},{"cell_type":"code","source":"def field__cross_analysis(field1, field2, num_round=1):\n    field1_nums = [col for col in NUMBER if col.startswith(field1)]\n    field2_nums = [col for col in NUMBER if col.startswith(field2)]\n    # D_num sum round 2\n    f, ax = plt.subplots(1,3,figsize=(12, 4))\n    plt.suptitle(f\"{field1} diff {field2} round {num_round}\")\n    tmp = train_last.groupby(\n        (train_last[field1_nums].mean(axis=1) - train_last[field2_nums].mean(axis=1)).round(num_round))\\\n    .target.mean().reset_index()\n    \n    sns.regplot(x=\"index\", y=\"target\", data=tmp, ax=ax[0])\n    sns.kdeplot(x=\"index\", y=\"target\", data=tmp, ax=ax[0])\n    ax[0].set_title(f\"mean diff vs. target\")\n    \n    tmp = train_last.groupby(\n        (train_last[field1_nums].std(axis=1) - train_last[field2_nums].std(axis=1)).round(num_round))\\\n    .target.mean().reset_index()\n    sns.regplot(x=\"index\", y=\"target\", data=tmp, ax=ax[1])\n    sns.kdeplot(x=\"index\", y=\"target\", data=tmp, ax=ax[1])\n    ax[1].set_title(f\"std diff vs. target\")\n    \n    tmp = train_last.groupby(\n        (train_last[field1_nums].isnull().sum(axis=1) - train_last[field2_nums].isnull().sum(axis=1)).round(num_round))\\\n    .target.mean().reset_index()\n\n    # f, ax = plt.subplots(figsize=(6, 6))\n    sns.regplot(x=\"index\", y=\"target\", data=tmp, ax=ax[2])\n    sns.kdeplot(x=\"index\", y=\"target\", data=tmp, ax=ax[2])\n    ax[2].set_title(f\"null sum diff vs. target\")","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:13:03.376614Z","iopub.execute_input":"2022-07-28T15:13:03.376970Z","iopub.status.idle":"2022-07-28T15:13:03.389286Z","shell.execute_reply.started":"2022-07-28T15:13:03.376940Z","shell.execute_reply":"2022-07-28T15:13:03.388382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import combinations","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:13:03.867061Z","iopub.execute_input":"2022-07-28T15:13:03.867368Z","iopub.status.idle":"2022-07-28T15:13:03.872179Z","shell.execute_reply.started":"2022-07-28T15:13:03.867340Z","shell.execute_reply":"2022-07-28T15:13:03.871071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for a, b in combinations([\"D\", \"P\", \"S\", \"B\", \"R\"], 2):\n    field__cross_analysis(a, b)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T15:13:04.350145Z","iopub.execute_input":"2022-07-28T15:13:04.350473Z","iopub.status.idle":"2022-07-28T15:13:24.051713Z","shell.execute_reply.started":"2022-07-28T15:13:04.350442Z","shell.execute_reply":"2022-07-28T15:13:24.050779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}