{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<font size=24>American Express - Default Prediction</font>","metadata":{"execution":{"iopub.status.busy":"2022-07-20T03:14:01.733396Z","iopub.execute_input":"2022-07-20T03:14:01.733732Z","iopub.status.idle":"2022-07-20T03:14:02.512710Z","shell.execute_reply.started":"2022-07-20T03:14:01.733704Z","shell.execute_reply":"2022-07-20T03:14:02.511760Z"}}},{"cell_type":"markdown","source":"<font size=3>由于数据集文件太过于庞大，因此首先对数据文件进行处理</font>","metadata":{}},{"cell_type":"markdown","source":"## 将标签与训练集对应","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:46:15.525977Z","iopub.execute_input":"2022-08-10T03:46:15.526471Z","iopub.status.idle":"2022-08-10T03:46:15.531966Z","shell.execute_reply.started":"2022-08-10T03:46:15.526428Z","shell.execute_reply":"2022-08-10T03:46:15.530748Z"}}},{"cell_type":"code","source":"pip install fastparquet","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport pandas as pd\ntrain_id=pd.read_csv('../input/tempresult/id.csv')\ntrain_label=pd.read_csv('../input/amex-default-prediction/train_labels.csv')\ntrain_id['label']=0\n\n#获取原标签中值为1的索引\nlabel_true=train_label[train_label['target']==1].index.tolist()\n\n#改变训练集id对应标签为1的标签为1\nlast_i=0\nindex=[0]\nfor i in label_true:\n    temp=(train_id.iloc[index[-1]+(i-last_i):index[-1]+(i-last_i)*13+13,0]==train_label.iloc[i,0])\n    last_i=i\n    index=temp[temp==True].index.tolist()\n    train_id.iloc[index,1]=1\n    \n#保存文件\ntrain_id.to_parquet('./train_id_label.parquet')\n'''\n#../input/aedptraindata/train_id_label.parquet","metadata":{"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 对训练集进行压缩","metadata":{}},{"cell_type":"code","source":"#pip install fastparquet","metadata":{"execution":{"iopub.status.busy":"2022-08-10T19:51:30.244499Z","iopub.execute_input":"2022-08-10T19:51:30.245236Z","iopub.status.idle":"2022-08-10T19:51:30.250305Z","shell.execute_reply.started":"2022-08-10T19:51:30.245197Z","shell.execute_reply":"2022-08-10T19:51:30.248851Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport pandas as pd\nimport fastparquet as fp\ndf = pd.read_csv(\"../input/amex-default-prediction/train_data.csv\", dtype = {'P_2':'float32','D_39':'float32','B_1':'float32','B_2':'float32','R_1':'float32','S_3':'float32','D_41':'float32','B_3':'float32','D_42':'float32','D_43':'float32','D_44':'float32','B_4':'float32','D_45':'float32','B_5':'float32','R_2':'float32','D_46':'float32','D_47':'float32','D_48':'float32','D_49':'float32','B_6':'float32','B_7':'float32','B_8':'float32','D_50':'float32','D_51':'float32','B_9':'float32','R_3':'float32','D_52':'float32','P_3':'float32','B_10':'float32','D_53':'float32','S_5':'float32','B_11':'float32','S_6':'float32','D_54':'float32','R_4':'float32','S_7':'float32','B_12':'float32','S_8':'float32','D_55':'float32','D_56':'float32','B_13':'float32','R_5':'float32','D_58':'float32','S_9':'float32','B_14':'float32','D_59':'float32','D_60':'float32','D_61':'float32','B_15':'float32','S_11':'float32','D_62':'float32','D_65':'float32','B_16':'float32','B_17':'float32','B_18':'float32','B_19':'float32','D_66':'float32','B_20':'float32','D_68':'float32','S_12':'float32','R_6':'float32','S_13':'float32','B_21':'float32','D_69':'float32','B_22':'float32','D_70':'float32','D_71':'float32','D_72':'float32','S_15':'float32','B_23':'float32','D_73':'float32','P_4':'float32','D_74':'float32','D_75':'float32','D_76':'float32','B_24':'float32','R_7':'float32','D_77':'float32','B_25':'float32','B_26':'float32','D_78':'float32','D_79':'float32','R_8':'float32','R_9':'float32','S_16':'float32','D_80':'float32','R_10':'float32','R_11':'float32','B_27':'float32','D_81':'float32','D_82':'float32','S_17':'float32','R_12':'float32','B_28':'float32','R_13':'float32','D_83':'float32','R_14':'float32','R_15':'float32','D_84':'float32','R_16':'float32','B_29':'float32','B_30':'float32','S_18':'float32','D_86':'float32','D_87':'float32','R_17':'float32','R_18':'float32','D_88':'float32','B_31':'float32','S_19':'float32','R_19':'float32','B_32':'float32','S_20':'float32','R_20':'float32','R_21':'float32','B_33':'float32','D_89':'float32','R_22':'float32','R_23':'float32','D_91':'float32','D_92':'float32','D_93':'float32','D_94':'float32','R_24':'float32','R_25':'float32','D_96':'float32','S_22':'float32','S_23':'float32','S_24':'float32','S_25':'float32','S_26':'float32','D_102':'float32','D_103':'float32','D_104':'float32','D_105':'float32','D_106':'float32','D_107':'float32','B_36':'float32','B_37':'float32','R_26':'float32','R_27':'float32','B_38':'float32','D_108':'float32','D_109':'float32','D_110':'float32','D_111':'float32','B_39':'float32','D_112':'float32','B_40':'float32','S_27':'float32','D_113':'float32','D_114':'float32','D_115':'float32','D_116':'float32','D_117':'float32','D_118':'float32','D_119':'float32','D_120':'float32','D_121':'float32','D_122':'float32','D_123':'float32','D_124':'float32','D_125':'float32','D_126':'float32','D_127':'float32','D_128':'float32','D_129':'float32','B_41':'float32','B_42':'float32','D_130':'float32','D_131':'float32','D_132':'float32','D_133':'float32','R_28':'float32','D_134':'float32','D_135':'float32','D_136':'float32','D_137':'float32','D_138':'float32','D_139':'float32','D_140':'float32','D_141':'float32','D_142':'float32','D_143':'float32','D_144':'float32','D_145':'float32'})\ndf.drop('customer_ID',axis=1,inplace=True)\ndf.to_parquet('./train_data.parquet',compression='brotli')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-10T19:51:30.625278Z","iopub.execute_input":"2022-08-10T19:51:30.626156Z","iopub.status.idle":"2022-08-10T19:51:30.634900Z","shell.execute_reply.started":"2022-08-10T19:51:30.626117Z","shell.execute_reply":"2022-08-10T19:51:30.634020Z"},"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 对测试集进行压缩","metadata":{}},{"cell_type":"code","source":"#pip install fastparquet","metadata":{"execution":{"iopub.status.busy":"2022-08-10T19:51:31.022370Z","iopub.execute_input":"2022-08-10T19:51:31.023058Z","iopub.status.idle":"2022-08-10T19:51:31.028231Z","shell.execute_reply.started":"2022-08-10T19:51:31.023009Z","shell.execute_reply":"2022-08-10T19:51:31.026907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-08-10T19:51:31.412011Z","iopub.execute_input":"2022-08-10T19:51:31.412447Z","iopub.status.idle":"2022-08-10T19:51:31.417570Z","shell.execute_reply.started":"2022-08-10T19:51:31.412411Z","shell.execute_reply":"2022-08-10T19:51:31.416172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntest1=pd.read_csv('../input/amex-default-prediction/test_data.csv',nrows=5000000, dtype = {'P_2':'float32','D_39':'float32','B_1':'float32','B_2':'float32','R_1':'float32','S_3':'float32','D_41':'float32','B_3':'float32','D_42':'float32','D_43':'float32','D_44':'float32','B_4':'float32','D_45':'float32','B_5':'float32','R_2':'float32','D_46':'float32','D_47':'float32','D_48':'float32','D_49':'float32','B_6':'float32','B_7':'float32','B_8':'float32','D_50':'float32','D_51':'float32','B_9':'float32','R_3':'float32','D_52':'float32','P_3':'float32','B_10':'float32','D_53':'float32','S_5':'float32','B_11':'float32','S_6':'float32','D_54':'float32','R_4':'float32','S_7':'float32','B_12':'float32','S_8':'float32','D_55':'float32','D_56':'float32','B_13':'float32','R_5':'float32','D_58':'float32','S_9':'float32','B_14':'float32','D_59':'float32','D_60':'float32','D_61':'float32','B_15':'float32','S_11':'float32','D_62':'float32','D_65':'float32','B_16':'float32','B_17':'float32','B_18':'float32','B_19':'float32','D_66':'float32','B_20':'float32','D_68':'float32','S_12':'float32','R_6':'float32','S_13':'float32','B_21':'float32','D_69':'float32','B_22':'float32','D_70':'float32','D_71':'float32','D_72':'float32','S_15':'float32','B_23':'float32','D_73':'float32','P_4':'float32','D_74':'float32','D_75':'float32','D_76':'float32','B_24':'float32','R_7':'float32','D_77':'float32','B_25':'float32','B_26':'float32','D_78':'float32','D_79':'float32','R_8':'float32','R_9':'float32','S_16':'float32','D_80':'float32','R_10':'float32','R_11':'float32','B_27':'float32','D_81':'float32','D_82':'float32','S_17':'float32','R_12':'float32','B_28':'float32','R_13':'float32','D_83':'float32','R_14':'float32','R_15':'float32','D_84':'float32','R_16':'float32','B_29':'float32','B_30':'float32','S_18':'float32','D_86':'float32','D_87':'float32','R_17':'float32','R_18':'float32','D_88':'float32','B_31':'float32','S_19':'float32','R_19':'float32','B_32':'float32','S_20':'float32','R_20':'float32','R_21':'float32','B_33':'float32','D_89':'float32','R_22':'float32','R_23':'float32','D_91':'float32','D_92':'float32','D_93':'float32','D_94':'float32','R_24':'float32','R_25':'float32','D_96':'float32','S_22':'float32','S_23':'float32','S_24':'float32','S_25':'float32','S_26':'float32','D_102':'float32','D_103':'float32','D_104':'float32','D_105':'float32','D_106':'float32','D_107':'float32','B_36':'float32','B_37':'float32','R_26':'float32','R_27':'float32','B_38':'float32','D_108':'float32','D_109':'float32','D_110':'float32','D_111':'float32','B_39':'float32','D_112':'float32','B_40':'float32','S_27':'float32','D_113':'float32','D_114':'float32','D_115':'float32','D_116':'float32','D_117':'float32','D_118':'float32','D_119':'float32','D_120':'float32','D_121':'float32','D_122':'float32','D_123':'float32','D_124':'float32','D_125':'float32','D_126':'float32','D_127':'float32','D_128':'float32','D_129':'float32','B_41':'float32','B_42':'float32','D_130':'float32','D_131':'float32','D_132':'float32','D_133':'float32','R_28':'float32','D_134':'float32','D_135':'float32','D_136':'float32','D_137':'float32','D_138':'float32','D_139':'float32','D_140':'float32','D_141':'float32','D_142':'float32','D_143':'float32','D_144':'float32','D_145':'float32'})\npd.DataFrame(test1['customer_ID']).to_parquet('./test_id1.parquet',compression='brotli')\ntest1.drop('customer_ID',axis=1,inplace=True)\ntest1.to_parquet('./test_data1.parquet',compression='brotli')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-10T19:51:31.803616Z","iopub.execute_input":"2022-08-10T19:51:31.804028Z","iopub.status.idle":"2022-08-10T19:51:31.813074Z","shell.execute_reply.started":"2022-08-10T19:51:31.803994Z","shell.execute_reply":"2022-08-10T19:51:31.811963Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns=['customer_ID', 'S_2', 'P_2', 'D_39', 'B_1', 'B_2', 'R_1', 'S_3',\n       'D_41', 'B_3', 'D_42', 'D_43', 'D_44', 'B_4', 'D_45', 'B_5', 'R_2',\n       'D_46', 'D_47', 'D_48', 'D_49', 'B_6', 'B_7', 'B_8', 'D_50',\n       'D_51', 'B_9', 'R_3', 'D_52', 'P_3', 'B_10', 'D_53', 'S_5', 'B_11',\n       'S_6', 'D_54', 'R_4', 'S_7', 'B_12', 'S_8', 'D_55', 'D_56', 'B_13',\n       'R_5', 'D_58', 'S_9', 'B_14', 'D_59', 'D_60', 'D_61', 'B_15',\n       'S_11', 'D_62', 'D_63', 'D_64', 'D_65', 'B_16', 'B_17', 'B_18',\n       'B_19', 'D_66', 'B_20', 'D_68', 'S_12', 'R_6', 'S_13', 'B_21',\n       'D_69', 'B_22', 'D_70', 'D_71', 'D_72', 'S_15', 'B_23', 'D_73',\n       'P_4', 'D_74', 'D_75', 'D_76', 'B_24', 'R_7', 'D_77', 'B_25',\n       'B_26', 'D_78', 'D_79', 'R_8', 'R_9', 'S_16', 'D_80', 'R_10',\n       'R_11', 'B_27', 'D_81', 'D_82', 'S_17', 'R_12', 'B_28', 'R_13',\n       'D_83', 'R_14', 'R_15', 'D_84', 'R_16', 'B_29', 'B_30', 'S_18',\n       'D_86', 'D_87', 'R_17', 'R_18', 'D_88', 'B_31', 'S_19', 'R_19',\n       'B_32', 'S_20', 'R_20', 'R_21', 'B_33', 'D_89', 'R_22', 'R_23',\n       'D_91', 'D_92', 'D_93', 'D_94', 'R_24', 'R_25', 'D_96', 'S_22',\n       'S_23', 'S_24', 'S_25', 'S_26', 'D_102', 'D_103', 'D_104', 'D_105',\n       'D_106', 'D_107', 'B_36', 'B_37', 'R_26', 'R_27', 'B_38', 'D_108',\n       'D_109', 'D_110', 'D_111', 'B_39', 'D_112', 'B_40', 'S_27',\n       'D_113', 'D_114', 'D_115', 'D_116', 'D_117', 'D_118', 'D_119',\n       'D_120', 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_126',\n       'D_127', 'D_128', 'D_129', 'B_41', 'B_42', 'D_130', 'D_131',\n       'D_132', 'D_133', 'R_28', 'D_134', 'D_135', 'D_136', 'D_137',\n       'D_138', 'D_139', 'D_140', 'D_141', 'D_142', 'D_143', 'D_144',\n       'D_145']","metadata":{"execution":{"iopub.status.busy":"2022-08-10T19:51:32.185483Z","iopub.execute_input":"2022-08-10T19:51:32.186292Z","iopub.status.idle":"2022-08-10T19:51:32.203948Z","shell.execute_reply.started":"2022-08-10T19:51:32.186228Z","shell.execute_reply":"2022-08-10T19:51:32.202616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntest2=pd.read_csv('../input/amex-default-prediction/test_data.csv',names=columns,skiprows=5000000, dtype = {'P_2':'float32','D_39':'float32','B_1':'float32','B_2':'float32','R_1':'float32','S_3':'float32','D_41':'float32','B_3':'float32','D_42':'float32','D_43':'float32','D_44':'float32','B_4':'float32','D_45':'float32','B_5':'float32','R_2':'float32','D_46':'float32','D_47':'float32','D_48':'float32','D_49':'float32','B_6':'float32','B_7':'float32','B_8':'float32','D_50':'float32','D_51':'float32','B_9':'float32','R_3':'float32','D_52':'float32','P_3':'float32','B_10':'float32','D_53':'float32','S_5':'float32','B_11':'float32','S_6':'float32','D_54':'float32','R_4':'float32','S_7':'float32','B_12':'float32','S_8':'float32','D_55':'float32','D_56':'float32','B_13':'float32','R_5':'float32','D_58':'float32','S_9':'float32','B_14':'float32','D_59':'float32','D_60':'float32','D_61':'float32','B_15':'float32','S_11':'float32','D_62':'float32','D_65':'float32','B_16':'float32','B_17':'float32','B_18':'float32','B_19':'float32','D_66':'float32','B_20':'float32','D_68':'float32','S_12':'float32','R_6':'float32','S_13':'float32','B_21':'float32','D_69':'float32','B_22':'float32','D_70':'float32','D_71':'float32','D_72':'float32','S_15':'float32','B_23':'float32','D_73':'float32','P_4':'float32','D_74':'float32','D_75':'float32','D_76':'float32','B_24':'float32','R_7':'float32','D_77':'float32','B_25':'float32','B_26':'float32','D_78':'float32','D_79':'float32','R_8':'float32','R_9':'float32','S_16':'float32','D_80':'float32','R_10':'float32','R_11':'float32','B_27':'float32','D_81':'float32','D_82':'float32','S_17':'float32','R_12':'float32','B_28':'float32','R_13':'float32','D_83':'float32','R_14':'float32','R_15':'float32','D_84':'float32','R_16':'float32','B_29':'float32','B_30':'float32','S_18':'float32','D_86':'float32','D_87':'float32','R_17':'float32','R_18':'float32','D_88':'float32','B_31':'float32','S_19':'float32','R_19':'float32','B_32':'float32','S_20':'float32','R_20':'float32','R_21':'float32','B_33':'float32','D_89':'float32','R_22':'float32','R_23':'float32','D_91':'float32','D_92':'float32','D_93':'float32','D_94':'float32','R_24':'float32','R_25':'float32','D_96':'float32','S_22':'float32','S_23':'float32','S_24':'float32','S_25':'float32','S_26':'float32','D_102':'float32','D_103':'float32','D_104':'float32','D_105':'float32','D_106':'float32','D_107':'float32','B_36':'float32','B_37':'float32','R_26':'float32','R_27':'float32','B_38':'float32','D_108':'float32','D_109':'float32','D_110':'float32','D_111':'float32','B_39':'float32','D_112':'float32','B_40':'float32','S_27':'float32','D_113':'float32','D_114':'float32','D_115':'float32','D_116':'float32','D_117':'float32','D_118':'float32','D_119':'float32','D_120':'float32','D_121':'float32','D_122':'float32','D_123':'float32','D_124':'float32','D_125':'float32','D_126':'float32','D_127':'float32','D_128':'float32','D_129':'float32','B_41':'float32','B_42':'float32','D_130':'float32','D_131':'float32','D_132':'float32','D_133':'float32','R_28':'float32','D_134':'float32','D_135':'float32','D_136':'float32','D_137':'float32','D_138':'float32','D_139':'float32','D_140':'float32','D_141':'float32','D_142':'float32','D_143':'float32','D_144':'float32','D_145':'float32'})\npd.DataFrame(test2['customer_ID']).to_parquet('./test_id2.parquet',compression='brotli')\ntest2.drop('customer_ID',axis=1,inplace=True)\ntest2.to_parquet('./test_data2.parquet',compression='brotli')\n'''\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T19:51:32.514794Z","iopub.execute_input":"2022-08-10T19:51:32.515184Z","iopub.status.idle":"2022-08-10T19:51:32.525200Z","shell.execute_reply.started":"2022-08-10T19:51:32.515154Z","shell.execute_reply":"2022-08-10T19:51:32.523723Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 导入数据","metadata":{}},{"cell_type":"code","source":"import  pandas as pd\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:11:06.683813Z","iopub.execute_input":"2022-08-12T08:11:06.685146Z","iopub.status.idle":"2022-08-12T08:11:06.709461Z","shell.execute_reply.started":"2022-08-12T08:11:06.685017Z","shell.execute_reply":"2022-08-12T08:11:06.708444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data=pd.read_parquet('../input/aedptraindata/train_data.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-08-12T08:11:48.133499Z","iopub.execute_input":"2022-08-12T08:11:48.133984Z","iopub.status.idle":"2022-08-12T08:12:30.044505Z","shell.execute_reply.started":"2022-08-12T08:11:48.133950Z","shell.execute_reply":"2022-08-12T08:12:30.043115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA(数据初步了解)","metadata":{}},{"cell_type":"code","source":"import  pandas as pd\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-08-10T19:51:34.333277Z","iopub.execute_input":"2022-08-10T19:51:34.334524Z","iopub.status.idle":"2022-08-10T19:51:34.339576Z","shell.execute_reply.started":"2022-08-10T19:51:34.334467Z","shell.execute_reply":"2022-08-10T19:51:34.338112Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:38:52.066034Z","iopub.execute_input":"2022-08-11T03:38:52.066641Z","iopub.status.idle":"2022-08-11T03:38:52.079649Z","shell.execute_reply.started":"2022-08-11T03:38:52.066587Z","shell.execute_reply":"2022-08-11T03:38:52.078349Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:38:52.082034Z","iopub.execute_input":"2022-08-11T03:38:52.083031Z","iopub.status.idle":"2022-08-11T03:38:52.120078Z","shell.execute_reply.started":"2022-08-11T03:38:52.082977Z","shell.execute_reply":"2022-08-11T03:38:52.118521Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_data.columns[train_data.head(100000).applymap(np.isreal).all(0)==0]","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:38:52.123411Z","iopub.execute_input":"2022-08-11T03:38:52.123927Z","iopub.status.idle":"2022-08-11T03:39:20.693562Z","shell.execute_reply.started":"2022-08-11T03:38:52.123888Z","shell.execute_reply":"2022-08-11T03:39:20.692418Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_data[[  'D_63', 'D_64']].describe(include=['O'])","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:20.694982Z","iopub.execute_input":"2022-08-11T03:39:20.695329Z","iopub.status.idle":"2022-08-11T03:39:21.982206Z","shell.execute_reply.started":"2022-08-11T03:39:20.695298Z","shell.execute_reply":"2022-08-11T03:39:21.981014Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_data[ 'D_63'].value_counts(),train_data[ 'D_64'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:21.983792Z","iopub.execute_input":"2022-08-11T03:39:21.984114Z","iopub.status.idle":"2022-08-11T03:39:22.435844Z","shell.execute_reply.started":"2022-08-11T03:39:21.984086Z","shell.execute_reply":"2022-08-11T03:39:22.434819Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font color='#ff590b'><font size=5>非数值属性4个,为['customer_ID', 'S_2', 'D_63', 'D_64']</font><br/>\n<font color='#ff590b'><font size=5>label为0和1，可以认为是一个二分类问题</font><br/>","metadata":{}},{"cell_type":"code","source":"#id=pd.read_csv('../input/tempresult/id.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:22.438105Z","iopub.execute_input":"2022-08-11T03:39:22.438904Z","iopub.status.idle":"2022-08-11T03:39:28.189106Z","shell.execute_reply.started":"2022-08-11T03:39:22.438860Z","shell.execute_reply":"2022-08-11T03:39:28.187924Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#id.value_counts().value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:39:28.193415Z","iopub.execute_input":"2022-08-11T03:39:28.193792Z","iopub.status.idle":"2022-08-11T03:39:30.554457Z","shell.execute_reply.started":"2022-08-11T03:39:28.193760Z","shell.execute_reply":"2022-08-11T03:39:30.553360Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport pandas as pd\nid_test1=pd.read_parquet('../input/aedp-test/test_id1.parquet')\nid_test2=pd.read_parquet('../input/aedp-test/test_id2.parquet')\ntest_id=pd.concat([id_test1, id_test2], ignore_index=True)\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-10T19:51:51.056219Z","iopub.execute_input":"2022-08-10T19:51:51.056735Z","iopub.status.idle":"2022-08-10T19:51:57.346584Z","shell.execute_reply.started":"2022-08-10T19:51:51.056686Z","shell.execute_reply":"2022-08-10T19:51:57.345305Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_id.value_counts().value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T19:51:57.348074Z","iopub.execute_input":"2022-08-10T19:51:57.348564Z","iopub.status.idle":"2022-08-10T19:52:02.039072Z","shell.execute_reply.started":"2022-08-10T19:51:57.348518Z","shell.execute_reply":"2022-08-10T19:52:02.037677Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font color='#ff590b'><font size=5>同一用户的记录条数最多的为13条</font><br/>","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:43:40.469560Z","iopub.execute_input":"2022-08-11T03:43:40.470178Z","iopub.status.idle":"2022-08-11T03:43:40.481619Z","shell.execute_reply.started":"2022-08-11T03:43:40.470133Z","shell.execute_reply":"2022-08-11T03:43:40.479780Z"}}},{"cell_type":"code","source":"#label=pd.read_csv('../input/amex-default-prediction/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:45:31.886385Z","iopub.execute_input":"2022-08-11T03:45:31.886886Z","iopub.status.idle":"2022-08-11T03:45:32.924482Z","shell.execute_reply.started":"2022-08-11T03:45:31.886846Z","shell.execute_reply":"2022-08-11T03:45:32.923326Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#label['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T03:45:35.541644Z","iopub.execute_input":"2022-08-11T03:45:35.542154Z","iopub.status.idle":"2022-08-11T03:45:35.558651Z","shell.execute_reply.started":"2022-08-11T03:45:35.542112Z","shell.execute_reply":"2022-08-11T03:45:35.556987Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font color='#ff590b'><font size=5>标签正样本与负样本比例为1:3</font><br/>","metadata":{}},{"cell_type":"markdown","source":"### 属性describe（）","metadata":{"execution":{"iopub.status.busy":"2022-08-10T10:32:54.254605Z","iopub.execute_input":"2022-08-10T10:32:54.254991Z","iopub.status.idle":"2022-08-10T10:32:54.259814Z","shell.execute_reply.started":"2022-08-10T10:32:54.254958Z","shell.execute_reply":"2022-08-10T10:32:54.258674Z"}}},{"cell_type":"code","source":"#获取训练集统计信息并保存\n'''\ndescribe=df.describe()\ndescribe.to_csv('./describe.csv')\n'''\n#../input/describe/describe.csv","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 缺失值与异常值","metadata":{}},{"cell_type":"code","source":"'''\nimport pandas as pd\ntrain_data=pd.read_parquet('../input/aedptraindata/train_data.parquet')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:29:36.623513Z","iopub.execute_input":"2022-08-11T14:29:36.624542Z","iopub.status.idle":"2022-08-11T14:30:14.998150Z","shell.execute_reply.started":"2022-08-11T14:29:36.624006Z","shell.execute_reply":"2022-08-11T14:30:14.996947Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#pd.DataFrame((train_data.isnull().sum()/len(train_data)).T).to_csv('./null.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-11T13:53:43.765030Z","iopub.execute_input":"2022-08-11T13:53:43.765657Z","iopub.status.idle":"2022-08-11T13:53:46.625255Z","shell.execute_reply.started":"2022-08-11T13:53:43.765606Z","shell.execute_reply":"2022-08-11T13:53:46.624048Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndes=train_data.describe()\nq1=des.iloc[4,1:]\nq3=des.iloc[6,1:]\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:38:28.112010Z","iopub.execute_input":"2022-08-11T14:38:28.113172Z","iopub.status.idle":"2022-08-11T14:39:23.730392Z","shell.execute_reply.started":"2022-08-11T14:38:28.113121Z","shell.execute_reply":"2022-08-11T14:39:23.727781Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\njudge=(train_data<q3+3*(q3-q1)) &(train_data>q1-3*(q3-q1))\n1-judge.sum()/len(train_data).to_csv('./1.csv')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T14:59:19.980663Z","iopub.execute_input":"2022-08-11T14:59:19.981243Z","iopub.status.idle":"2022-08-11T14:59:34.956087Z","shell.execute_reply.started":"2022-08-11T14:59:19.981202Z","shell.execute_reply":"2022-08-11T14:59:34.954706Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 属性间相关性","metadata":{"execution":{"iopub.status.busy":"2022-08-11T05:15:05.101023Z","iopub.execute_input":"2022-08-11T05:15:05.101616Z","iopub.status.idle":"2022-08-11T05:15:05.108400Z","shell.execute_reply.started":"2022-08-11T05:15:05.101561Z","shell.execute_reply":"2022-08-11T05:15:05.106964Z"}}},{"cell_type":"code","source":"'''\nimport pandas as pd\nid_label=pd.read_parquet('../input/aedptraindata/train_id_label.parquet')\ntrain_data=pd.read_parquet('../input/aedptraindata/train_data.parquet')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:00:48.232085Z","iopub.execute_input":"2022-08-11T16:00:48.233232Z","iopub.status.idle":"2022-08-11T16:01:39.356618Z","shell.execute_reply.started":"2022-08-11T16:00:48.233090Z","shell.execute_reply":"2022-08-11T16:01:39.352415Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntrain_data['label']=0\ntrain_data['label'].iloc[:100000]=id_label.iloc[:100000,1]\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:01:39.361882Z","iopub.execute_input":"2022-08-11T16:01:39.362868Z","iopub.status.idle":"2022-08-11T16:01:39.625421Z","shell.execute_reply.started":"2022-08-11T16:01:39.362788Z","shell.execute_reply":"2022-08-11T16:01:39.624054Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimportance=train_data.iloc[:100000].corr()['label']\nimportance.to_csv('p.csv')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:02:02.541761Z","iopub.execute_input":"2022-08-11T16:02:02.542201Z","iopub.status.idle":"2022-08-11T16:02:10.429490Z","shell.execute_reply.started":"2022-08-11T16:02:02.542168Z","shell.execute_reply":"2022-08-11T16:02:10.428172Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimportance=train_data.iloc[:40000].corr(method='kendall')['label']\nimportance.to_csv('k.csv')\n'''\n","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimportance=train_data.iloc[:100000].corr(method='spearman')['label']\nimportance.to_csv('s.csv')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:09:25.063696Z","iopub.execute_input":"2022-08-11T16:09:25.064125Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### id重排","metadata":{}},{"cell_type":"code","source":"'''\ntrain_ID=pd.read_parquet('../input/aedptraindata/train_id_label.parquet')\ntrain_ID['new_id']=0\n\n\nindex=[-1]\nfor j in range(458913-1):\n    temp=train_ID.iloc[index[-1]+1:index[-1]+1+13,0]==train_ID.iloc[index[-1]+1,0]\n    index=temp[temp==True].index.tolist()\n    train_ID.iloc[index,1]=j\n    \ntrain_ID.iloc[index[-1]+1:,1]=458913\ntrain_ID.to_parquet('./train_newid.parquet')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T10:20:01.634726Z","iopub.execute_input":"2022-08-11T10:20:01.635204Z","iopub.status.idle":"2022-08-11T10:30:14.502809Z","shell.execute_reply.started":"2022-08-11T10:20:01.635163Z","shell.execute_reply":"2022-08-11T10:30:14.501073Z"},"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport pandas as pd\nid_test1=pd.read_parquet('../input/aedp-test/test_id1.parquet')\nid_test2=pd.read_parquet('../input/aedp-test/test_id2.parquet')\ntest_id=pd.concat([id_test1, id_test2], ignore_index=True)\n\ntest_id['new_id']=0\n\nindex=[-1]\nfor j in range(1000000):\n    temp=test_id.iloc[index[-1]+1:index[-1]+1+13,0]==test_id.iloc[index[-1]+1,0]\n    index=temp[temp==True].index.tolist()\n    test_id.iloc[index,1]=j\n    if(j%1000==0):\n        test_id.to_parquet('./test_newid.parquet')\n        print(index[-1])\n    if(index[-1]==11363754):\n        break\n#test_id.iloc[index[-1]+1:,1]=j+1\ntest_id.to_parquet('./test_newid.parquet')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-11T10:35:30.173701Z","iopub.execute_input":"2022-08-11T10:35:30.174192Z","iopub.status.idle":"2022-08-11T10:35:37.116756Z","shell.execute_reply.started":"2022-08-11T10:35:30.174156Z","shell.execute_reply":"2022-08-11T10:35:37.115397Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 数据预处理","metadata":{}},{"cell_type":"markdown","source":"### 属性名处理","metadata":{}},{"cell_type":"code","source":"'''\ncolumns=train_data.columns.values\ntrain_data.columns.values\n'''","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#获取读取时数据类型\n'''\nstring=''\nfor i in range(len(train_data.columns.values)):\n    string+='\\''\n    string=string+columns[i]+'\\':\\'float32\\','\n'''\nstring=\"'customer_ID':str,'S_2':str,'P_2':'float32','D_39':'float32','B_1':'float32','B_2':'float32','R_1':'float32','S_3':'float32','D_41':'float32','B_3':'float32','D_42':'float32','D_43':'float32','D_44':'float32','B_4':'float32','D_45':'float32','B_5':'float32','R_2':'float32','D_46':'float32','D_47':'float32','D_48':'float32','D_49':'float32','B_6':'float32','B_7':'float32','B_8':'float32','D_50':'float32','D_51':'float32','B_9':'float32','R_3':'float32','D_52':'float32','P_3':'float32','B_10':'float32','D_53':'float32','S_5':'float32','B_11':'float32','S_6':'float32','D_54':'float32','R_4':'float32','S_7':'float32','B_12':'float32','S_8':'float32','D_55':'float32','D_56':'float32','B_13':'float32','R_5':'float32','D_58':'float32','S_9':'float32','B_14':'float32','D_59':'float32','D_60':'float32','D_61':'float32','B_15':'float32','S_11':'float32','D_62':'float32','D_63':str,'D_64':str,'D_65':'float32','B_16':'float32','B_17':'float32','B_18':'float32','B_19':'float32','D_66':'float32','B_20':'float32','D_68':'float32','S_12':'float32','R_6':'float32','S_13':'float32','B_21':'float32','D_69':'float32','B_22':'float32','D_70':'float32','D_71':'float32','D_72':'float32','S_15':'float32','B_23':'float32','D_73':'float32','P_4':'float32','D_74':'float32','D_75':'float32','D_76':'float32','B_24':'float32','R_7':'float32','D_77':'float32','B_25':'float32','B_26':'float32','D_78':'float32','D_79':'float32','R_8':'float32','R_9':'float32','S_16':'float32','D_80':'float32','R_10':'float32','R_11':'float32','B_27':'float32','D_81':'float32','D_82':'float32','S_17':'float32','R_12':'float32','B_28':'float32','R_13':'float32','D_83':'float32','R_14':'float32','R_15':'float32','D_84':'float32','R_16':'float32','B_29':'float32','B_30':'float32','S_18':'float32','D_86':'float32','D_87':'float32','R_17':'float32','R_18':'float32','D_88':'float32','B_31':'float32','S_19':'float32','R_19':'float32','B_32':'float32','S_20':'float32','R_20':'float32','R_21':'float32','B_33':'float32','D_89':'float32','R_22':'float32','R_23':'float32','D_91':'float32','D_92':'float32','D_93':'float32','D_94':'float32','R_24':'float32','R_25':'float32','D_96':'float32','S_22':'float32','S_23':'float32','S_24':'float32','S_25':'float32','S_26':'float32','D_102':'float32','D_103':'float32','D_104':'float32','D_105':'float32','D_106':'float32','D_107':'float32','B_36':'float32','B_37':'float32','R_26':'float32','R_27':'float32','B_38':'float32','D_108':'float32','D_109':'float32','D_110':'float32','D_111':'float32','B_39':'float32','D_112':'float32','B_40':'float32','S_27':'float32','D_113':'float32','D_114':'float32','D_115':'float32','D_116':'float32','D_117':'float32','D_118':'float32','D_119':'float32','D_120':'float32','D_121':'float32','D_122':'float32','D_123':'float32','D_124':'float32','D_125':'float32','D_126':'float32','D_127':'float32','D_128':'float32','D_129':'float32','B_41':'float32','B_42':'float32','D_130':'float32','D_131':'float32','D_132':'float32','D_133':'float32','R_28':'float32','D_134':'float32','D_135':'float32','D_136':'float32','D_137':'float32','D_138':'float32','D_139':'float32','D_140':'float32','D_141':'float32','D_142':'float32','D_143':'float32','D_144':'float32','D_145':'float32'\"\n","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 变量提取","metadata":{}},{"cell_type":"code","source":"col=['B_1','B_10','B_11','B_12','B_16','B_17','B_18','B_19','B_2','B_20','B_21','B_22','B_23','B_24','B_25','B_28','B_3','B_30','B_31','B_32','B_33','B_36','B_37','B_38','B_4','B_40','B_41','B_42','B_5','B_6','B_7','B_8','B_9','D_103','D_104','D_107','D_109','D_112','D_113','D_114','D_115','D_116','D_117','D_118','D_119','D_120','D_121','D_122','D_123','D_124','D_125','D_127','D_128','D_129','D_130','D_131','D_132','D_133','D_139','D_140','D_141','D_142','D_143','D_145','D_41','D_42','D_43','D_44','D_45','D_46','D_47','D_48','D_50','D_51','D_52','D_53','D_54','D_55','D_56','D_58','D_61','D_62','D_65','D_68','D_70','D_71','D_72','D_74','D_75','D_76','D_77','D_78','D_79','D_80','D_81','D_83','D_84','D_86','D_87','D_89','D_91','D_92','D_93','D_94','D_96','P_2','P_3','P_4','R_1','R_10','R_11','R_12','R_13','R_14','R_15','R_16','R_17','R_19','R_2','R_20','R_21','R_22','R_24','R_25','R_26','R_27','R_28','R_3','R_4','R_5','R_6','R_7','R_8','S_15','S_16','S_20','S_23','S_25','S_26','S_3','S_5','S_7','S_9']\n#90个属性\nlen(col)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T02:54:00.804848Z","iopub.execute_input":"2022-08-13T02:54:00.805598Z","iopub.status.idle":"2022-08-13T02:54:00.852607Z","shell.execute_reply.started":"2022-08-13T02:54:00.805467Z","shell.execute_reply":"2022-08-13T02:54:00.851364Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport pandas as pd\ntrain_data=pd.read_parquet('../input/aedptraindata/train_data.parquet')\ntrain_label=pd.read_parquet('../input/aedptraindata/train_id_label.parquet')\ntrain_id=pd.read_parquet('../input/aedptraindata/train_newid.parquet')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:03:56.164892Z","iopub.execute_input":"2022-08-13T05:03:56.165224Z","iopub.status.idle":"2022-08-13T05:04:43.112220Z","shell.execute_reply.started":"2022-08-13T05:03:56.165153Z","shell.execute_reply":"2022-08-13T05:04:43.110955Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndata=train_data[['B_1','B_10','B_11','B_12','B_16','B_17','B_18','B_19','B_2','B_20','B_21','B_22','B_23','B_24','B_25','B_28','B_3','B_30','B_31','B_32','B_33','B_36','B_37','B_38','B_4','B_40','B_41','B_42','B_5','B_6','B_7','B_8','B_9','D_103','D_104','D_107','D_109','D_112','D_113','D_114','D_115','D_116','D_117','D_118','D_119','D_120','D_121','D_122','D_123','D_124','D_125','D_127','D_128','D_129','D_130','D_131','D_132','D_133','D_139','D_140','D_141','D_142','D_143','D_145','D_41','D_42','D_43','D_44','D_45','D_46','D_47','D_48','D_50','D_51','D_52','D_53','D_54','D_55','D_56','D_58','D_61','D_62','D_63','D_64','D_65','D_68','D_70','D_71','D_72','D_74','D_75','D_76','D_77','D_78','D_79','D_80','D_81','D_83','D_84','D_86','D_87','D_89','D_91','D_92','D_93','D_94','D_96','P_2','P_3','P_4','R_1','R_10','R_11','R_12','R_13','R_14','R_15','R_16','R_17','R_19','R_2','R_20','R_21','R_22','R_24','R_25','R_26','R_27','R_28','R_3','R_4','R_5','R_6','R_7','R_8','S_15','S_16','S_20','S_23','S_25','S_26','S_3','S_5','S_7','S_9']]\nid=train_id['label']\nid.name='id'\nlabel=train_label['label']\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:04:57.929724Z","iopub.execute_input":"2022-08-13T05:04:57.930606Z","iopub.status.idle":"2022-08-13T05:04:58.622208Z","shell.execute_reply.started":"2022-08-13T05:04:57.930580Z","shell.execute_reply":"2022-08-13T05:04:58.621055Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lstm_data=pd.concat([id,data,label],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:05:01.195417Z","iopub.execute_input":"2022-08-13T05:05:01.195759Z","iopub.status.idle":"2022-08-13T05:05:03.256605Z","shell.execute_reply.started":"2022-08-13T05:05:01.195733Z","shell.execute_reply":"2022-08-13T05:05:03.255913Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport gc\ndel train_data\ngc.collect()\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-13T13:06:24.777774Z","iopub.execute_input":"2022-08-13T13:06:24.778672Z","iopub.status.idle":"2022-08-13T13:06:24.786628Z","shell.execute_reply.started":"2022-08-13T13:06:24.778625Z","shell.execute_reply":"2022-08-13T13:06:24.785450Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## IDNRNN","metadata":{}},{"cell_type":"markdown","source":"### 数据预处理","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nlstm_data=pd.read_parquet('../input/aedp-lstm-train-data/lstm_data/lstm_data.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-08-18T13:55:37.797108Z","iopub.execute_input":"2022-08-18T13:55:37.797810Z","iopub.status.idle":"2022-08-18T13:56:05.644923Z","shell.execute_reply.started":"2022-08-18T13:55:37.797707Z","shell.execute_reply":"2022-08-18T13:56:05.643933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in ['B_1','B_10','B_11','B_12','B_16','B_17','B_18','B_19','B_2','B_20','B_21','B_22','B_23','B_24','B_25','B_28','B_3','B_30','B_31','B_32','B_33','B_36','B_37','B_38','B_4','B_40','B_41','B_42','B_5','B_6','B_7','B_8','B_9','D_103','D_104','D_107','D_109','D_112','D_113','D_114','D_115','D_116','D_117','D_118','D_119','D_120','D_121','D_122','D_123','D_124','D_125','D_127','D_128','D_129','D_130','D_131','D_132','D_133','D_139','D_140','D_141','D_142','D_143','D_145','D_41','D_42','D_43','D_44','D_45','D_46','D_47','D_48','D_50','D_51','D_52','D_53','D_54','D_55','D_56','D_58','D_61','D_62','D_65','D_68','D_70','D_71','D_72','D_74','D_75','D_76','D_77','D_78','D_79','D_80','D_81','D_83','D_84','D_86','D_87','D_89','D_91','D_92','D_93','D_94','D_96','P_2','P_3','P_4','R_1','R_10','R_11','R_12','R_13','R_14','R_15','R_16','R_17','R_19','R_2','R_20','R_21','R_22','R_24','R_25','R_26','R_27','R_28','R_3','R_4','R_5','R_6','R_7','R_8','S_15','S_16','S_20','S_23','S_25','S_26','S_3','S_5','S_7','S_9']:\n    lstm_data.loc[lstm_data[i].isna(),i]=lstm_data.loc[lstm_data[i].isna(),'label'].map({1: lstm_data.loc[lstm_data['label']==1,i].mean(),0: lstm_data.loc[lstm_data['label']==0,i].mean() })\nlstm_data['D_63'] = lstm_data['D_63'] .map({'CO': 0.15,'CR': 0.3,'CL':0.45,'XZ':0.6,'XM':0.75,'XL':0.9 })\nlstm_data['D_64'] = lstm_data['D_64'] .map({'O': 0.3,'U': 0.6,'R':0.9 })   \nlstm_data['D_64']=lstm_data['D_64'].fillna(lstm_data.loc[lstm_data['D_64'].notna(),'D_64'].mean())","metadata":{"execution":{"iopub.status.busy":"2022-08-18T13:56:05.647120Z","iopub.execute_input":"2022-08-18T13:56:05.647991Z","iopub.status.idle":"2022-08-18T13:56:32.270157Z","shell.execute_reply.started":"2022-08-18T13:56:05.647944Z","shell.execute_reply":"2022-08-18T13:56:32.268978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 网络搭建","metadata":{}},{"cell_type":"markdown","source":"### 训练","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch.nn import Parameter\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nimport math\n\nclass IndRNNCell_onlyrecurrent(nn.Module):\n\n    def __init__(self, hidden_size, \n                 hidden_max_abs=None, recurrent_init=None):\n        super(IndRNNCell_onlyrecurrent, self).__init__()\n        self.hidden_size = hidden_size\n        self.recurrent_init = recurrent_init\n        self.weight_hh = Parameter(torch.Tensor(hidden_size))            \n        self.reset_parameters()\n\n    def reset_parameters(self):\n        for name, weight in self.named_parameters():\n            if \"weight_hh\" in name:\n                if self.recurrent_init is None:\n                    nn.init.uniform(weight, a=0, b=1)\n                else:\n                    self.recurrent_init(weight)\n\n    def forward(self, input, hx):\n        return F.relu(input + hx * self.weight_hh.unsqueeze(0).expand(hx.size(0), len(self.weight_hh)))\n\n\nclass IndRNN_onlyrecurrent(nn.Module):\n    def __init__(self, hidden_size,recurrent_init=None, **kwargs):\n        super(IndRNN_onlyrecurrent, self).__init__()\n        self.hidden_size = hidden_size\n        self.indrnn_cell=IndRNNCell_onlyrecurrent(hidden_size, **kwargs)\n\n        if recurrent_init is not None:\n            kwargs[\"recurrent_init\"] = recurrent_init\n        self.recurrent_init=recurrent_init\n        # h0 = torch.zeros(hidden_size * num_directions)\n        # self.register_buffer('h0', torch.autograd.Variable(h0))\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        for name, weight in self.named_parameters():\n            if \"weight_hh\" in name:\n                if self.recurrent_init is None:\n                    nn.init.uniform(weight, a=0, b=1)\n                else:\n                    self.recurrent_init(weight)\n\n    def forward(self, input, h0=None):\n        assert input.dim() == 2 or input.dim() == 3        \n        if h0 is None:\n            h0 = input.data.new(input.size(-2),input.size(-1)).zero_().contiguous()\n        elif (h0.size(-1)!=input.size(-1)) or (h0.size(-2)!=input.size(-2)):\n            raise RuntimeError(\n                'The initial hidden size must be equal to input_size. Expected {}, got {}'.format(\n                    h0.size(), input.size()))\n        outputs=[]\n        hx_cell=h0\n        for input_t in input:\n            hx_cell = self.indrnn_cell(input_t, hx_cell)\n            outputs.append(hx_cell)\n        out_put = torch.stack(outputs, 0)\n        return out_put","metadata":{"execution":{"iopub.status.busy":"2022-08-18T13:56:32.271953Z","iopub.execute_input":"2022-08-18T13:56:32.272384Z","iopub.status.idle":"2022-08-18T13:56:34.210506Z","shell.execute_reply.started":"2022-08-18T13:56:32.272341Z","shell.execute_reply":"2022-08-18T13:56:34.209095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset,DataLoader\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport time\ndevice= torch.device(\"cuda:0\" if torch.cuda. is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-08-18T13:56:34.213394Z","iopub.execute_input":"2022-08-18T13:56:34.214044Z","iopub.status.idle":"2022-08-18T13:56:34.219913Z","shell.execute_reply.started":"2022-08-18T13:56:34.214003Z","shell.execute_reply":"2022-08-18T13:56:34.218584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1,14):\n    if(i==13):\n        batchsize=64\n        inputdim=13\n        epoches=10\n    elif(i<7):\n        batchsize=8\n        inputdim=i\n        epoches=100\n    elif(i<11 and i>6):\n        batchsize=16\n        inputdim=i\n        epoches=100\n    elif(i<13 and i>10):\n        batchsize=32\n        inputdim=i\n        epoches=100\n        \n    class Net(nn.Module):\n        def __init__(self,input_dim,batchsize):\n            super(Net, self).__init__()\n            self.hidden_size = 9\n            self.input_dim=inputdim\n            self.batchsize = batchsize\n\n\n            self.connection = nn.Sequential(\n                nn.Linear(145, 72),\n                nn.ReLU(True),\n                nn.BatchNorm1d(self.input_dim),\n\n                nn.Linear(72, 36),\n                nn.ReLU(True),\n                nn.BatchNorm1d(self.input_dim),\n\n                nn.Linear(36, 18),\n                nn.ReLU(True),\n                nn.BatchNorm1d(self.input_dim),\n\n                nn.Linear(18, 9),\n                nn.BatchNorm1d(self.input_dim),\n            )\n\n            self.idnrnn=IndRNN_onlyrecurrent(hidden_size=9)\n\n            self.prob=nn.Sequential(\n                    nn.Linear(9, 1),\n            )\n\n\n        def forward(self, x,h0):\n            x = self.connection(x)\n            for i in range(self.input_dim):\n                h0 = self.idnrnn(x[:,i,:].view(1,self.batchsize,9),h0)[0]\n            h=self.prob(h0)\n            return h\n        \n\n    class ListDataset(Dataset):\n        def __init__(self,inputdim):\n            self.intputdim=inputdim\n            self.group=lstm_data.groupby('id').groups\n            self.index1=lstm_data.groupby('id').size()[lstm_data.groupby('id').size()==self.intputdim]\n\n        def __getitem__(self, index):\n            return torch.tensor(np.array(lstm_data.iloc[self.group[self.index1.index[index]].values,1:-1])).to(torch.float32).to(device),torch.tensor(np.array(lstm_data.iloc[self.index1.index[index],-1])).to(torch.float32).to(device)\n\n        def __len__(self):\n            return len(self.index1)\n        \n    dataset = ListDataset(inputdim)\n    dataloader = torch.utils.data.DataLoader(dataset,batch_size=batchsize,drop_last=True)\n    model13=Net(inputdim,batchsize)\n    optimizer=torch.optim.Adam(model13.parameters(),lr=0.0003)\n    model13.to(device)\n    \n    h0=torch.zeros(batchsize,9).to(device)\n    start_time = time.time()\n    model13.train()\n    for epoch in range(epoches):\n        losses=[]\n        for a,(b,c) in enumerate(dataloader):\n            optimizer.zero_grad()\n            results = model13(b,h0)\n            criterion = nn.BCEWithLogitsLoss()\n            loss=criterion(results, c.view(len(c),1))\n            losses.append(loss)\n            loss.backward()\n            optimizer.step()\n            if(a%1000==1):\n                havetime=time.time()-start_time\n                print('model:',i,'epoch',epoch,'/',epoches,havetime,sum(losses)/len(losses),len(dataloader)*epoches/a*havetime-havetime)\n                exec (\"torch.save(model13, './model%s.model')\"%i)\n    exec (\"torch.save(model13, './model%s.model')\"%i)","metadata":{"execution":{"iopub.status.busy":"2022-08-18T14:24:09.665089Z","iopub.execute_input":"2022-08-18T14:24:09.665545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 预测","metadata":{}},{"cell_type":"code","source":"'''\nfrom torch.utils.data import Dataset,DataLoader\nimport torch\nimport torch.nn as nn\nimport numpy as np\nimport time\ndevice= torch.device(\"cuda:0\" if torch.cuda. is_available() else \"cpu\")\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-18T13:08:50.013300Z","iopub.execute_input":"2022-08-18T13:08:50.013772Z","iopub.status.idle":"2022-08-18T13:08:50.587579Z","shell.execute_reply.started":"2022-08-18T13:08:50.013735Z","shell.execute_reply":"2022-08-18T13:08:50.586378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport pandas as pd\ntest1=pd.read_parquet('../input/aedp-test/test/test_data1.parquet')\n#test2=pd.read_parquet('../input/aedp-test/test/test_data2.parquet')\n#test2.drop(0,axis=0)\ntest_id = pd.read_parquet('../input/aedp-test/test_newid.parquet')\nid=test_id.iloc[:5000000]\n#id=test_id.iloc[5000002:]\nsample_submission=pd.read_csv('../input/amex-default-prediction/sample_submission.csv')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-18T13:08:52.015466Z","iopub.execute_input":"2022-08-18T13:08:52.016168Z","iopub.status.idle":"2022-08-18T13:09:34.027501Z","shell.execute_reply.started":"2022-08-18T13:08:52.016127Z","shell.execute_reply":"2022-08-18T13:09:34.026259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimport torch\nfrom torch.nn import Parameter\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nimport math\n\nclass IndRNNCell_onlyrecurrent(nn.Module):\n\n    def __init__(self, hidden_size, \n                 hidden_max_abs=None, recurrent_init=None):\n        super(IndRNNCell_onlyrecurrent, self).__init__()\n        self.hidden_size = hidden_size\n        self.recurrent_init = recurrent_init\n        self.weight_hh = Parameter(torch.Tensor(hidden_size))            \n        self.reset_parameters()\n\n    def reset_parameters(self):\n        for name, weight in self.named_parameters():\n            if \"weight_hh\" in name:\n                if self.recurrent_init is None:\n                    nn.init.uniform(weight, a=0, b=1)\n                else:\n                    self.recurrent_init(weight)\n\n    def forward(self, input, hx):\n        return F.relu(input + hx * self.weight_hh.unsqueeze(0).expand(hx.size(0), len(self.weight_hh)))\n\n\nclass IndRNN_onlyrecurrent(nn.Module):\n    def __init__(self, hidden_size,recurrent_init=None, **kwargs):\n        super(IndRNN_onlyrecurrent, self).__init__()\n        self.hidden_size = hidden_size\n        self.indrnn_cell=IndRNNCell_onlyrecurrent(hidden_size, **kwargs)\n\n        if recurrent_init is not None:\n            kwargs[\"recurrent_init\"] = recurrent_init\n        self.recurrent_init=recurrent_init\n        # h0 = torch.zeros(hidden_size * num_directions)\n        # self.register_buffer('h0', torch.autograd.Variable(h0))\n        self.reset_parameters()\n\n    def reset_parameters(self):\n        for name, weight in self.named_parameters():\n            if \"weight_hh\" in name:\n                if self.recurrent_init is None:\n                    nn.init.uniform(weight, a=0, b=1)\n                else:\n                    self.recurrent_init(weight)\n\n    def forward(self, input, h0=None):\n        assert input.dim() == 2 or input.dim() == 3        \n        if h0 is None:\n            h0 = input.data.new(input.size(-2),input.size(-1)).zero_().contiguous()\n        elif (h0.size(-1)!=input.size(-1)) or (h0.size(-2)!=input.size(-2)):\n            raise RuntimeError(\n                'The initial hidden size must be equal to input_size. Expected {}, got {}'.format(\n                    h0.size(), input.size()))\n        outputs=[]\n        hx_cell=h0\n        for input_t in input:\n            hx_cell = self.indrnn_cell(input_t, hx_cell)\n            outputs.append(hx_cell)\n        out_put = torch.stack(outputs, 0)\n        return out_put\n\nclass Net(nn.Module):\n        def __init__(self,input_dim,batchsize):\n            super(Net, self).__init__()\n            self.hidden_size = 9\n            self.input_dim=inputdim\n            self.batchsize = batchsize\n\n\n            self.connection = nn.Sequential(\n                nn.Linear(145, 72),\n                nn.ReLU(True),\n                nn.BatchNorm1d(self.input_dim),\n\n                nn.Linear(72, 36),\n                nn.ReLU(True),\n                nn.BatchNorm1d(self.input_dim),\n\n                nn.Linear(36, 18),\n                nn.ReLU(True),\n                nn.BatchNorm1d(self.input_dim),\n\n                nn.Linear(18, 9),\n                nn.BatchNorm1d(self.input_dim),\n            )\n\n            self.idnrnn=IndRNN_onlyrecurrent(hidden_size=9)\n\n            self.prob=nn.Sequential(\n                    nn.Linear(9, 1),\n            )\n\n\n        def forward(self, x,h0):\n            x = self.connection(x)\n            for i in range(self.input_dim):\n                h0 = self.idnrnn(x[:,i,:].view(1,self.batchsize,9),h0)[0]\n            h=self.prob(h0)\n            return h\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-18T12:53:01.702906Z","iopub.execute_input":"2022-08-18T12:53:01.703795Z","iopub.status.idle":"2022-08-18T12:53:01.730117Z","shell.execute_reply.started":"2022-08-18T12:53:01.703743Z","shell.execute_reply":"2022-08-18T12:53:01.729070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test=test1[['B_1','B_10','B_11','B_12','B_16','B_17','B_18','B_19','B_2','B_20','B_21','B_22','B_23','B_24','B_25','B_28','B_3','B_30','B_31','B_32','B_33','B_36','B_37','B_38','B_4','B_40','B_41','B_42','B_5','B_6','B_7','B_8','B_9','D_103','D_104','D_107','D_109','D_112','D_113','D_114','D_115','D_116','D_117','D_118','D_119','D_120','D_121','D_122','D_123','D_124','D_125','D_127','D_128','D_129','D_130','D_131','D_132','D_133','D_139','D_140','D_141','D_142','D_143','D_145','D_41','D_42','D_43','D_44','D_45','D_46','D_47','D_48','D_50','D_51','D_52','D_53','D_54','D_55','D_56','D_58','D_61','D_62','D_63','D_64','D_65','D_68','D_70','D_71','D_72','D_74','D_75','D_76','D_77','D_78','D_79','D_80','D_81','D_83','D_84','D_86','D_87','D_89','D_91','D_92','D_93','D_94','D_96','P_2','P_3','P_4','R_1','R_10','R_11','R_12','R_13','R_14','R_15','R_16','R_17','R_19','R_2','R_20','R_21','R_22','R_24','R_25','R_26','R_27','R_28','R_3','R_4','R_5','R_6','R_7','R_8','S_15','S_16','S_20','S_23','S_25','S_26','S_3','S_5','S_7','S_9']]\n#test=test2[['B_1','B_10','B_11','B_12','B_16','B_17','B_18','B_19','B_2','B_20','B_21','B_22','B_23','B_24','B_25','B_28','B_3','B_30','B_31','B_32','B_33','B_36','B_37','B_38','B_4','B_40','B_41','B_42','B_5','B_6','B_7','B_8','B_9','D_103','D_104','D_107','D_109','D_112','D_113','D_114','D_115','D_116','D_117','D_118','D_119','D_120','D_121','D_122','D_123','D_124','D_125','D_127','D_128','D_129','D_130','D_131','D_132','D_133','D_139','D_140','D_141','D_142','D_143','D_145','D_41','D_42','D_43','D_44','D_45','D_46','D_47','D_48','D_50','D_51','D_52','D_53','D_54','D_55','D_56','D_58','D_61','D_62','D_63','D_64','D_65','D_68','D_70','D_71','D_72','D_74','D_75','D_76','D_77','D_78','D_79','D_80','D_81','D_83','D_84','D_86','D_87','D_89','D_91','D_92','D_93','D_94','D_96','P_2','P_3','P_4','R_1','R_10','R_11','R_12','R_13','R_14','R_15','R_16','R_17','R_19','R_2','R_20','R_21','R_22','R_24','R_25','R_26','R_27','R_28','R_3','R_4','R_5','R_6','R_7','R_8','S_15','S_16','S_20','S_23','S_25','S_26','S_3','S_5','S_7','S_9']]","metadata":{"execution":{"iopub.status.busy":"2022-08-18T12:53:01.733885Z","iopub.execute_input":"2022-08-18T12:53:01.734730Z","iopub.status.idle":"2022-08-18T12:53:04.344218Z","shell.execute_reply.started":"2022-08-18T12:53:01.734663Z","shell.execute_reply":"2022-08-18T12:53:04.343139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndescribe=pd.read_csv('../input/describe/describe.csv')\ndescribe=describe.T\nmean=describe.iloc[1:,1][['B_1','B_10','B_11','B_12','B_16','B_17','B_18','B_19','B_2','B_20','B_21','B_22','B_23','B_24','B_25','B_28','B_3','B_30','B_31','B_32','B_33','B_36','B_37','B_38','B_4','B_40','B_41','B_42','B_5','B_6','B_7','B_8','B_9','D_103','D_104','D_107','D_109','D_112','D_113','D_114','D_115','D_116','D_117','D_118','D_119','D_120','D_121','D_122','D_123','D_124','D_125','D_127','D_128','D_129','D_130','D_131','D_132','D_133','D_139','D_140','D_141','D_142','D_143','D_145','D_41','D_42','D_43','D_44','D_45','D_46','D_47','D_48','D_50','D_51','D_52','D_53','D_54','D_55','D_56','D_58','D_61','D_62','D_65','D_68','D_70','D_71','D_72','D_74','D_75','D_76','D_77','D_78','D_79','D_80','D_81','D_83','D_84','D_86','D_87','D_89','D_91','D_92','D_93','D_94','D_96','P_2','P_3','P_4','R_1','R_10','R_11','R_12','R_13','R_14','R_15','R_16','R_17','R_19','R_2','R_20','R_21','R_22','R_24','R_25','R_26','R_27','R_28','R_3','R_4','R_5','R_6','R_7','R_8','S_15','S_16','S_20','S_23','S_25','S_26','S_3','S_5','S_7','S_9']]\nmean\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-18T12:53:04.346984Z","iopub.execute_input":"2022-08-18T12:53:04.348473Z","iopub.status.idle":"2022-08-18T12:53:04.390637Z","shell.execute_reply.started":"2022-08-18T12:53:04.348418Z","shell.execute_reply":"2022-08-18T12:53:04.389092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntest['D_63'] = test['D_63'] .map({'CO': 0.15,'CR': 0.3,'CL':0.45,'XZ':0.6,'XM':0.75,'XL':0.9 })\ntest['D_64'] =test['D_64'] .map({'O': 0.3,'U': 0.6,'R':0.9 })   \ntest['D_64']=test['D_64'].fillna(test.loc[test['D_64'].notna(),'D_64'].mean())\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-18T12:53:04.393018Z","iopub.execute_input":"2022-08-18T12:53:04.394273Z","iopub.status.idle":"2022-08-18T12:53:05.422296Z","shell.execute_reply.started":"2022-08-18T12:53:04.394231Z","shell.execute_reply":"2022-08-18T12:53:05.420970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test.fillna(mean)","metadata":{"execution":{"iopub.status.busy":"2022-08-18T12:53:05.423750Z","iopub.execute_input":"2022-08-18T12:53:05.424156Z","iopub.status.idle":"2022-08-18T12:53:11.643585Z","shell.execute_reply.started":"2022-08-18T12:53:05.424123Z","shell.execute_reply":"2022-08-18T12:53:11.642524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nbatchsize=128\nh0=torch.zeros(batchsize,9)\nfor i in range(1,14):\n    exec (\"model%s=torch.load('../input/aedp-lstm-train-data/model/model%d.model')\"%(i,i))\n    inputdim=i\n    h0=torch\n    class ListDataset(Dataset):\n        def __init__(self,inputdim):\n            self.intputdim=inputdim\n            self.group=id.groupby('new_id').groups\n            self.index1=id.groupby('new_id').size()[id.groupby('new_id').size()==self.intputdim]\n\n        def __getitem__(self, index):\n            return torch.tensor(np.array(test.iloc[self.group[self.index1.index[index]].values,:])),self.index1.index[index]\n\n        def __len__(self):\n            return len(self.index1)\n        \n    dataset = ListDataset(inputdim)\n    dataloader = torch.utils.data.DataLoader(dataset,batch_size=batchsize)\n    \n    for a,(b,c) in enumerate(dataloader):\n        exec(\"sample_submission[c]=model%s(b,h0)\"%i)\n'''","metadata":{"execution":{"iopub.status.busy":"2022-08-18T13:02:36.207230Z","iopub.execute_input":"2022-08-18T13:02:36.207647Z","iopub.status.idle":"2022-08-18T13:02:42.392931Z","shell.execute_reply.started":"2022-08-18T13:02:36.207615Z","shell.execute_reply":"2022-08-18T13:02:42.391117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}