{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import libraries and load data from feather files","metadata":{}},{"cell_type":"code","source":"!pip install deepctr_torch","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport sys\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfrom plotly.offline import init_notebook_mode\nimport cudf\nimport cupy\nimport gc\n\n\nimport torch\nfrom deepctr_torch.inputs import SparseFeat, DenseFeat, get_feature_names\nfrom deepctr_torch.models import *\nfrom tqdm import tqdm\nfrom sklearn.model_selection import GridSearchCV\nimport math\nplt.style.use('ggplot')\nimport warnings as w\nw.filterwarnings(action='ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-01T04:29:47.148739Z","iopub.execute_input":"2022-07-01T04:29:47.149938Z","iopub.status.idle":"2022-07-01T04:29:56.032562Z","shell.execute_reply.started":"2022-07-01T04:29:47.149901Z","shell.execute_reply":"2022-07-01T04:29:56.031101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns',None)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:29:56.034049Z","iopub.execute_input":"2022-07-01T04:29:56.034927Z","iopub.status.idle":"2022-07-01T04:29:56.039753Z","shell.execute_reply.started":"2022-07-01T04:29:56.034887Z","shell.execute_reply":"2022-07-01T04:29:56.038323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_feather('../input/amexfeather/train_data.ftr')\ntrain = train.groupby('customer_ID').tail(1).set_index('customer_ID')\nprint(\"The training data begins on {} and ends on {}.\".format(train['S_2'].min().strftime('%m-%d-%Y'),train['S_2'].max().strftime('%m-%d-%Y')))\nprint(\"There are {:,.0f} customers in the training set and {} features.\".format(train.shape[0],train.shape[1]))\n\ntest = pd.read_feather('../input/amexfeather/test_data.ftr')\ntest = test.groupby('customer_ID').tail(1).set_index('customer_ID')\nprint(\"\\nThe test data begins on {} and ends on {}.\".format(test['S_2'].min().strftime('%m-%d-%Y'),test['S_2'].max().strftime('%m-%d-%Y')))\nprint(\"There are {:,.0f} customers in the test set and {} features.\".format(test.shape[0],test.shape[1]))\n\ncategorical_feature = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_68', 'D_64', 'D_66']\ndel test['S_2']\ndel train['S_2']\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:29:56.041805Z","iopub.execute_input":"2022-07-01T04:29:56.042142Z","iopub.status.idle":"2022-07-01T04:31:04.551354Z","shell.execute_reply.started":"2022-07-01T04:29:56.042105Z","shell.execute_reply":"2022-07-01T04:31:04.550651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:04.552375Z","iopub.execute_input":"2022-07-01T04:31:04.552878Z","iopub.status.idle":"2022-07-01T04:31:04.699512Z","shell.execute_reply.started":"2022-07-01T04:31:04.552841Z","shell.execute_reply":"2022-07-01T04:31:04.698566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:04.700864Z","iopub.execute_input":"2022-07-01T04:31:04.701246Z","iopub.status.idle":"2022-07-01T04:31:04.932876Z","shell.execute_reply.started":"2022-07-01T04:31:04.701205Z","shell.execute_reply":"2022-07-01T04:31:04.932044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Explain\n 1. D_* = Delinquency Variable (criminal?)\n 2. S_* = Spend Varibale \n 3. P_* = Payment Variable\n 4. B_* = Balance Variable\n 5. R_* = Risk variable\n \n### Categorical Variable\n   * 'B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68'","metadata":{}},{"cell_type":"markdown","source":"# EDA skip in this notebook\n#### Link: https://www.kaggle.com/code/leejunseok97/amex-default-eda-prediction","metadata":{}},{"cell_type":"markdown","source":"# Features Labeling","metadata":{}},{"cell_type":"code","source":"numeric_feature = [cols for cols in train.columns if cols not in categorical_feature]\nfor feature in numeric_feature:\n    if train[feature][0].dtype == np.float16:\n        train[feature].fillna(-99.0,inplace=True)\n        test[feature].fillna(-99.0,inplace=True)\n    else:\n        pass\ntrain.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:04.934404Z","iopub.execute_input":"2022-07-01T04:31:04.937758Z","iopub.status.idle":"2022-07-01T04:31:06.572325Z","shell.execute_reply.started":"2022-07-01T04:31:04.937718Z","shell.execute_reply":"2022-07-01T04:31:06.571602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nencoder = LabelEncoder()\ncategorical_feature = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_68', 'D_64', 'D_66']\ncolumns = train.columns.values\nfor feature in categorical_feature:\n    if feature in columns:\n        train[feature] = encoder.fit_transform(train[feature])\n        test[feature] = encoder.fit_transform(test[feature])\n    else:\n        pass","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:06.573479Z","iopub.execute_input":"2022-07-01T04:31:06.574955Z","iopub.status.idle":"2022-07-01T04:31:08.111095Z","shell.execute_reply.started":"2022-07-01T04:31:06.574916Z","shell.execute_reply":"2022-07-01T04:31:08.110248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dense_feature = [cols for cols in train.columns if cols not in categorical_feature]\ndense_feature.remove('target')\nsparse_feature = categorical_feature\ntest_dense_feature = [cols for cols in test.columns if cols not in categorical_feature]\ntest_sparse_feature = categorical_feature\ntarget = ['target']\nprint('Dense Feature:',dense_feature)\nprint('-'*58)\nprint('sparse_feature:',sparse_feature)\nprint('-'*58)\nprint('Target:',target)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:08.112256Z","iopub.execute_input":"2022-07-01T04:31:08.112812Z","iopub.status.idle":"2022-07-01T04:31:08.121725Z","shell.execute_reply.started":"2022-07-01T04:31:08.112773Z","shell.execute_reply":"2022-07-01T04:31:08.120568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dense_feature in sparse_feature","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:08.124675Z","iopub.execute_input":"2022-07-01T04:31:08.125158Z","iopub.status.idle":"2022-07-01T04:31:08.136667Z","shell.execute_reply.started":"2022-07-01T04:31:08.125107Z","shell.execute_reply":"2022-07-01T04:31:08.135644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fixlen_feature_columns = [SparseFeat(feat,train[feat].nunique())\n                          for feat in sparse_feature] + [DenseFeat(feat,1) for feat in dense_feature]\nfixlen_feature_columns","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:08.138242Z","iopub.execute_input":"2022-07-01T04:31:08.138630Z","iopub.status.idle":"2022-07-01T04:31:08.180197Z","shell.execute_reply.started":"2022-07-01T04:31:08.138592Z","shell.execute_reply":"2022-07-01T04:31:08.179488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dnn_feature_columns = fixlen_feature_columns\nlinear_feature_columns = fixlen_feature_columns","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:08.181329Z","iopub.execute_input":"2022-07-01T04:31:08.181810Z","iopub.status.idle":"2022-07-01T04:31:08.185604Z","shell.execute_reply.started":"2022-07-01T04:31:08.181775Z","shell.execute_reply":"2022-07-01T04:31:08.184638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:08.186896Z","iopub.execute_input":"2022-07-01T04:31:08.187490Z","iopub.status.idle":"2022-07-01T04:31:08.194554Z","shell.execute_reply.started":"2022-07-01T04:31:08.187454Z","shell.execute_reply":"2022-07-01T04:31:08.193738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model_input = {name:train[name] for name in feature_names}\ntest_model_input = {name:test[name] for name in feature_names}","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:08.195792Z","iopub.execute_input":"2022-07-01T04:31:08.196222Z","iopub.status.idle":"2022-07-01T04:31:08.216969Z","shell.execute_reply.started":"2022-07-01T04:31:08.196187Z","shell.execute_reply":"2022-07-01T04:31:08.216137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model testing","metadata":{}},{"cell_type":"code","source":"device = ('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:08.218204Z","iopub.execute_input":"2022-07-01T04:31:08.218571Z","iopub.status.idle":"2022-07-01T04:31:08.227906Z","shell.execute_reply.started":"2022-07-01T04:31:08.218536Z","shell.execute_reply":"2022-07-01T04:31:08.227046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = xDeepFM(linear_feature_columns,dnn_feature_columns,task='binary',\n                device=device)\n# model = AutoInt(linear_feature_columns,dnn_feature_columns,task='binary',device=device)\n# model = DIFM(linear_feature_columns,dnn_feature_columns,task='binary',device=device)\nmodel","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:08.229147Z","iopub.execute_input":"2022-07-01T04:31:08.229631Z","iopub.status.idle":"2022-07-01T04:31:14.363568Z","shell.execute_reply.started":"2022-07-01T04:31:08.229596Z","shell.execute_reply":"2022-07-01T04:31:14.362608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer=torch.optim.AdamW(model.parameters(),lr=3e-4, weight_decay=1e-2), # 3e-4\n    loss='binary_crossentropy',\n    metrics=['binary_crossentropy','auc']\n)\nhistory = model.fit(train_model_input, train[target].values, batch_size=1024,\n                    epochs=70,verbose=1,validation_split=0.2)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:31:14.364933Z","iopub.execute_input":"2022-07-01T04:31:14.365374Z","iopub.status.idle":"2022-07-01T04:42:58.610914Z","shell.execute_reply.started":"2022-07-01T04:31:14.365336Z","shell.execute_reply":"2022-07-01T04:42:58.609981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Export prediction","metadata":{}},{"cell_type":"code","source":"pred = model.predict(test_model_input,1024)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:42:58.612352Z","iopub.execute_input":"2022-07-01T04:42:58.612872Z","iopub.status.idle":"2022-07-01T04:43:10.880896Z","shell.execute_reply.started":"2022-07-01T04:42:58.612833Z","shell.execute_reply":"2022-07-01T04:43:10.880077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:43:10.882169Z","iopub.execute_input":"2022-07-01T04:43:10.882598Z","iopub.status.idle":"2022-07-01T04:43:12.565352Z","shell.execute_reply.started":"2022-07-01T04:43:10.882559Z","shell.execute_reply":"2022-07-01T04:43:12.564537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['prediction'] = pred\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:43:12.566668Z","iopub.execute_input":"2022-07-01T04:43:12.567204Z","iopub.status.idle":"2022-07-01T04:43:12.579988Z","shell.execute_reply.started":"2022-07-01T04:43:12.567141Z","shell.execute_reply":"2022-07-01T04:43:12.579024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-01T04:43:12.581221Z","iopub.execute_input":"2022-07-01T04:43:12.581557Z","iopub.status.idle":"2022-07-01T04:43:17.453458Z","shell.execute_reply.started":"2022-07-01T04:43:12.581523Z","shell.execute_reply":"2022-07-01T04:43:17.452594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}