{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-27T09:47:09.343140Z","iopub.execute_input":"2022-07-27T09:47:09.343724Z","iopub.status.idle":"2022-07-27T09:47:09.393088Z","shell.execute_reply.started":"2022-07-27T09:47:09.343613Z","shell.execute_reply":"2022-07-27T09:47:09.391902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib.ticker import MaxNLocator\nfrom matplotlib.colors import ListedColormap\nfrom cycler import cycler\nfrom IPython.display import display\nimport datetime\nimport scipy.stats\nimport warnings\nfrom colorama import Fore, Back, Style\nimport gc\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom lightgbm import LGBMClassifier\nimport lightgbm as lgb\n\nplt.rcParams['axes.facecolor'] = '#0057b8' # blue\nplt.rcParams['axes.prop_cycle'] = cycler(color=['#ffd700'] +\n                                         plt.rcParams['axes.prop_cycle'].by_key()['color'][1:])\nplt.rcParams['text.color'] = 'w'\n\nINFERENCE = True # set to False if you only want to cross-validate\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:47:10.006066Z","iopub.execute_input":"2022-07-27T09:47:10.006896Z","iopub.status.idle":"2022-07-27T09:47:12.309084Z","shell.execute_reply.started":"2022-07-27T09:47:10.006843Z","shell.execute_reply":"2022-07-27T09:47:12.308097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_parquet(f'../input/amex-fe/train_fe_plus_plus.parquet')\ntest=pd.read_parquet(f'../input/amex-fe/test_fe_plus_plus.parquet')\n#target = pd.read_csv('../input/amex-default-prediction/train_labels.csv').target.values\n\n#print(f\"target shape: {target.shape}\")\nprint(f'Train data memory usage: {train.memory_usage().sum() / 1e9} GBytes')\nprint(f'Test data memory usage:  {test.memory_usage().sum() / 1e9} GBytes')\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:47:12.311208Z","iopub.execute_input":"2022-07-27T09:47:12.311970Z","iopub.status.idle":"2022-07-27T09:48:02.467663Z","shell.execute_reply.started":"2022-07-27T09:47:12.311920Z","shell.execute_reply":"2022-07-27T09:48:02.466416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T08:45:37.304819Z","iopub.execute_input":"2022-07-27T08:45:37.305183Z","iopub.status.idle":"2022-07-27T08:45:37.447294Z","shell.execute_reply.started":"2022-07-27T08:45:37.305153Z","shell.execute_reply":"2022-07-27T08:45:37.446109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T08:45:37.449415Z","iopub.execute_input":"2022-07-27T08:45:37.449833Z","iopub.status.idle":"2022-07-27T08:45:37.458865Z","shell.execute_reply.started":"2022-07-27T08:45:37.449790Z","shell.execute_reply":"2022-07-27T08:45:37.457967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"miss_train_test=[ 'B_29_', 'S_9_','D_87_','D_88_', 'R_26_', 'R_27_', 'D_108_', 'D_110_', 'D_111_', 'B_39_', 'B_42_']","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:53:28.415285Z","iopub.execute_input":"2022-07-27T09:53:28.416153Z","iopub.status.idle":"2022-07-27T09:53:28.421468Z","shell.execute_reply.started":"2022-07-27T09:53:28.416107Z","shell.execute_reply":"2022-07-27T09:53:28.420524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"D41=[ c for c in train.columns if 'S_23_' in c]\nprint(D41)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T08:01:06.751026Z","iopub.execute_input":"2022-07-27T08:01:06.751518Z","iopub.status.idle":"2022-07-27T08:01:06.759546Z","shell.execute_reply.started":"2022-07-27T08:01:06.751475Z","shell.execute_reply":"2022-07-27T08:01:06.757760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# total=pd.concat([train,test])","metadata":{"execution":{"iopub.status.busy":"2022-07-27T06:20:33.407303Z","iopub.execute_input":"2022-07-27T06:20:33.407777Z","iopub.status.idle":"2022-07-27T06:20:39.520477Z","shell.execute_reply.started":"2022-07-27T06:20:33.407739Z","shell.execute_reply":"2022-07-27T06:20:39.519400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scale_shift=['D_41_','D_121_','B_37_','S_25_', 'S_23_', 'B_1_', 'S_11_','B_29_']","metadata":{"execution":{"iopub.status.busy":"2022-07-27T08:45:37.460160Z","iopub.execute_input":"2022-07-27T08:45:37.460485Z","iopub.status.idle":"2022-07-27T08:45:37.469365Z","shell.execute_reply.started":"2022-07-27T08:45:37.460449Z","shell.execute_reply":"2022-07-27T08:45:37.467919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.set_style(style='white')\n# plt.figure(figsize=(50,50))\n# for j,item in enumerate(miss_train_test):\n#     tmp=[ c for c in total.columns if item in c]\n#     for i, f in enumerate(tmp):\n#         plt.subplot(17, 4, j*6+i+1)\n#         sns.kdeplot(train[f])\n#         sns.kdeplot(test[f])\n        \n    \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-27T08:45:37.471303Z","iopub.execute_input":"2022-07-27T08:45:37.472351Z","iopub.status.idle":"2022-07-27T08:45:37.480410Z","shell.execute_reply.started":"2022-07-27T08:45:37.472312Z","shell.execute_reply":"2022-07-27T08:45:37.479269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsns.set_style(style='white')\nfor j,item in enumerate(scale_shift):\n    plt.figure(figsize=(15,15))\n    tmp=[ c for c in train.columns if item in c]\n    for i, f in enumerate(tmp):\n        plt.subplot(3, 3, i+1)\n        sns.kdeplot(train[f], color='red',  fill=True)\n        sns.kdeplot(test[f], color='green', fill=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T08:52:31.260093Z","iopub.execute_input":"2022-07-27T08:52:31.260572Z","iopub.status.idle":"2022-07-27T08:57:29.664004Z","shell.execute_reply.started":"2022-07-27T08:52:31.260539Z","shell.execute_reply":"2022-07-27T08:57:29.662974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsns.set_style(style='white')\nfor j,item in enumerate(['D_121_','D_124_','B_39_','D_142_','R_5_','D_59_','D_50_','D_61_']):\n    plt.figure(figsize=(15,15))\n    tmp=[ c for c in train.columns if item in c]\n    for i, f in enumerate(tmp):\n        plt.subplot(3, 3, i+1)\n        sns.kdeplot(train[f], color='red',  fill=True)\n        sns.kdeplot(test[f], color='green', fill=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:08:39.268089Z","iopub.execute_input":"2022-07-27T09:08:39.269314Z","iopub.status.idle":"2022-07-27T09:12:27.962481Z","shell.execute_reply.started":"2022-07-27T09:08:39.269273Z","shell.execute_reply":"2022-07-27T09:12:27.961234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del total\n#gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T06:52:49.344236Z","iopub.execute_input":"2022-07-27T06:52:49.344680Z","iopub.status.idle":"2022-07-27T06:52:49.725616Z","shell.execute_reply.started":"2022-07-27T06:52:49.344644Z","shell.execute_reply":"2022-07-27T06:52:49.724357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tmp_col_shift=['B_37_last','S_11_min','S_11_last','S_11_mean','S_11_max','S_11_last_lag_div']\n# plt.figure(figsize=(20,20))\n# for i,c in enumerate(tmp_col_shift):\n#     plt.subplot(len(tmp_col_shift),2,i*2+1)\n#     norm_c_train=(train[c]-train[c].mean())/(train[c].std())\n#     norm_c_test=(test[c]-test[c].mean())/(test[c].std())\n#     sns.kdeplot(train[c], fill=True, color='red')\n#     sns.kdeplot(test[c], fill=True,color='blue')\n    \n#     plt.subplot(len(tmp_col_shift),2, i*2+2)\n#     sns.kdeplot(norm_c_train, fill=True, color='red')\n#     sns.kdeplot(norm_c_test, fill=True,color='blue')\n    \n#     del norm_c_train, norm_c_test\n#     gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:01:01.156029Z","iopub.execute_input":"2022-07-27T09:01:01.156421Z","iopub.status.idle":"2022-07-27T09:01:01.161678Z","shell.execute_reply.started":"2022-07-27T09:01:01.156391Z","shell.execute_reply":"2022-07-27T09:01:01.160543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tmp=[c for c in train.columns if 'B_39_' in c]\n# train[tmp].describe()\n# #print(tmp)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:59:51.540153Z","iopub.execute_input":"2022-07-27T09:59:51.540661Z","iopub.status.idle":"2022-07-27T09:59:51.696608Z","shell.execute_reply.started":"2022-07-27T09:59:51.540627Z","shell.execute_reply":"2022-07-27T09:59:51.695132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test[tmp].describe()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:59:58.059987Z","iopub.execute_input":"2022-07-27T09:59:58.060472Z","iopub.status.idle":"2022-07-27T09:59:58.323399Z","shell.execute_reply.started":"2022-07-27T09:59:58.060435Z","shell.execute_reply":"2022-07-27T09:59:58.322332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"miss_train_test=[ 'B_29_', 'S_9_','D_87_','D_88_', 'R_26_', 'R_27_', 'D_108_', 'D_110_', 'D_111_', 'B_39_', 'B_42_']\nscale_shift=['D_41_','D_123_','B_37_','S_25_', 'S_23_', 'B_1_', 'S_11_','B_29_']\nnew_scale=['D_121_','D_124_','D_142_','R_5_','D_59_','D_50_','D_61_','S_9_'] + scale_shift\ndrop=list(set(miss_train_test).intersection(new_scale))\n\nprint(drop)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T10:05:34.711966Z","iopub.execute_input":"2022-07-27T10:05:34.712453Z","iopub.status.idle":"2022-07-27T10:05:34.720464Z","shell.execute_reply.started":"2022-07-27T10:05:34.712421Z","shell.execute_reply":"2022-07-27T10:05:34.719280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_col=[c for i in drop for c in train.columns if i in c]","metadata":{"execution":{"iopub.status.busy":"2022-07-27T10:13:13.837110Z","iopub.execute_input":"2022-07-27T10:13:13.837893Z","iopub.status.idle":"2022-07-27T10:13:13.844178Z","shell.execute_reply.started":"2022-07-27T10:13:13.837848Z","shell.execute_reply":"2022-07-27T10:13:13.843034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[drop_col].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T10:13:44.946865Z","iopub.execute_input":"2022-07-27T10:13:44.947939Z","iopub.status.idle":"2022-07-27T10:13:45.393020Z","shell.execute_reply.started":"2022-07-27T10:13:44.947878Z","shell.execute_reply":"2022-07-27T10:13:45.391890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[drop_col].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T10:14:37.211699Z","iopub.execute_input":"2022-07-27T10:14:37.212215Z","iopub.status.idle":"2022-07-27T10:14:38.290031Z","shell.execute_reply.started":"2022-07-27T10:14:37.212174Z","shell.execute_reply":"2022-07-27T10:14:38.288847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n#bin_features=['B_31_', 'D_87_']","metadata":{"execution":{"iopub.status.busy":"2022-07-27T10:06:36.991716Z","iopub.execute_input":"2022-07-27T10:06:36.992220Z","iopub.status.idle":"2022-07-27T10:06:36.998298Z","shell.execute_reply.started":"2022-07-27T10:06:36.992181Z","shell.execute_reply":"2022-07-27T10:06:36.997120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#bin_all=[c for c in train.columns if bin_features[0] in c or bin_features[1] in c]\n#print(bin_all)\n#test[bin_all].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-27T10:11:49.870701Z","iopub.execute_input":"2022-07-27T10:11:49.871093Z","iopub.status.idle":"2022-07-27T10:11:49.875580Z","shell.execute_reply.started":"2022-07-27T10:11:49.871061Z","shell.execute_reply":"2022-07-27T10:11:49.874485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_plus_data(df):\n    scale_shift=[]\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_cat=pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet',columns=cat_features+['customer_ID']+bin_features)","metadata":{"execution":{"iopub.status.busy":"2022-07-27T09:40:57.955242Z","iopub.execute_input":"2022-07-27T09:40:57.955626Z","iopub.status.idle":"2022-07-27T09:41:00.670233Z","shell.execute_reply.started":"2022-07-27T09:40:57.955597Z","shell.execute_reply":"2022-07-27T09:41:00.669377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_cat.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop(columns=drop_col,inplace=True)\ntest.drop(columns=drop_col,inplace=True)\n","metadata":{},"execution_count":null,"outputs":[]}]}