{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-14T05:36:36.765969Z","iopub.execute_input":"2022-09-14T05:36:36.766428Z","iopub.status.idle":"2022-09-14T05:36:36.804825Z","shell.execute_reply.started":"2022-09-14T05:36:36.766331Z","shell.execute_reply":"2022-09-14T05:36:36.803369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Data Preprosessing","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport gc\nfrom sklearn.preprocessing import OrdinalEncoder\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Using feather data format instead of csv can speed up the I/O and save the working memory. \n# https://towardsdatascience.com/the-best-format-to-save-pandas-data-414dca023e0d\n# Some dataset includes only the last month feature with dropping all the other 12 rows,\n# because the last row contains the most important information according to this article.\n# https://www.kaggle.com/code/eteresh/amex-nn-proof-last-features-enough\n# Data trasforming with this notebook:\n# https://www.kaggle.com/code/ragnar123/amex-lgbm-dart-cv-0-7977/notebook\n\ntrain_dataset = pd.read_feather('../input/amexfeather/train_data.ftr')\nprint(train_dataset.shape)\ntrain_dataset.info()","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:36:36.808397Z","iopub.execute_input":"2022-09-14T05:36:36.809413Z","iopub.status.idle":"2022-09-14T05:37:04.466183Z","shell.execute_reply.started":"2022-09-14T05:36:36.809371Z","shell.execute_reply":"2022-09-14T05:37:04.464068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data exploration","metadata":{}},{"cell_type":"code","source":"Y = train_dataset[['customer_ID','target']].set_index('customer_ID', drop=True)\n# plt.hist(Y.values)\nY['true_false'] = Y['target']==1.0\n# Y.head(5)\n# Y[Y.index == \"ffffa5c46bc8de74f5a4554e74e239c8dee6b9baf388145b2c3d01967fcce461\"]\ncustomer_Y = Y.groupby('customer_ID')['target'].count().rename('count').to_frame()\n# customer_Y\ncustomer_Y[\"true_count\"] = Y.groupby(['customer_ID'])['true_false'].sum()\n# customer_Y\n# customer_Y[customer_Y[\"true_count\"]>1]\n# customer_Y.loc[customer_Y.index == \"ffffa5c46bc8de74f5a4554e74e239c8dee6b9baf388145b2c3d01967fcce461\",]\ncustomer_Y['percent'] = customer_Y['true_count']/customer_Y['count'] * 100\nplt.hist(customer_Y['percent'],bins = 10,align= 'mid')\n# plt.legend()\nplt.xlabel(\"percentage %\")\nplt.ylabel(\"counts\")\nplt.title(\"Percentage of true targets for each customer_ID\")\nplt.text(12, 330000, r'0%')\nplt.text(90, 130000, r'100%')\n# It proves that for all customer_IDs there is only a final target","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:37:04.468772Z","iopub.execute_input":"2022-09-14T05:37:04.469334Z","iopub.status.idle":"2022-09-14T05:37:08.113813Z","shell.execute_reply.started":"2022-09-14T05:37:04.469284Z","shell.execute_reply":"2022-09-14T05:37:08.112422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = Y.reset_index().groupby('customer_ID').tail(1).set_index('customer_ID', drop=True).sort_index()\ny['true_false'].value_counts().plot(kind='bar')\nplt.xlabel(\"defaults\")\nplt.ylabel(\"numbers\")\nplt.title(\"Number of defaults\")\na = y['true_false'].value_counts().values\nplt.text(0.3,300000, a[0])\nplt.text(1, 130000, a[1])\n\ny = y.drop(\"true_false\",axis = 1)\n# save label\ny.to_csv(\"./label.csv\")\n\ndel Y\ndel customer_Y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:37:08.117847Z","iopub.execute_input":"2022-09-14T05:37:08.118324Z","iopub.status.idle":"2022-09-14T05:37:11.398141Z","shell.execute_reply.started":"2022-09-14T05:37:08.118289Z","shell.execute_reply":"2022-09-14T05:37:11.396473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n# plt.figure(figsize=(12, 8))\n# sns.distplot(train_dataset_['B_2'], bins=500, kde=False)\n\n# plt.figure(figsize=(12, 8))\n# sns.distplot(train_dataset_.loc[train_dataset_.B_2<0.2,'B_2'],bins=500,kde=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:37:11.400096Z","iopub.execute_input":"2022-09-14T05:37:11.400502Z","iopub.status.idle":"2022-09-14T05:37:11.605117Z","shell.execute_reply.started":"2022-09-14T05:37:11.400465Z","shell.execute_reply":"2022-09-14T05:37:11.603766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig = plt.figure(figsize=(16, 10))\n# sns.kdeplot(train_dataset['P_2'], log_scale=[False,True])\n# sns.kdeplot(train_dataset.loc[train_dataset.target == 1,'P_2'], log_scale=[False,True])\n# sns.kdeplot(train_dataset.loc[train_dataset.target == 0,'P_2'], log_scale=[False,True])\n# fig.legend(labels=['First','Second','Third'])","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:37:11.606802Z","iopub.execute_input":"2022-09-14T05:37:11.607199Z","iopub.status.idle":"2022-09-14T05:37:11.613362Z","shell.execute_reply.started":"2022-09-14T05:37:11.607164Z","shell.execute_reply":"2022-09-14T05:37:11.611997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(9, 9))\nsns.kdeplot(train_dataset['D_43'], log_scale=[False,True])\nsns.kdeplot(train_dataset.loc[train_dataset.target == 1,'D_43'], log_scale=[False,True])\nsns.kdeplot(train_dataset.loc[train_dataset.target == 0,'D_43'], log_scale=[False,True])\nfig.legend(labels=['Full Sample','Default','Good'])\nplt.title(\"Comparison of Feature D_43\")","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:37:11.615206Z","iopub.execute_input":"2022-09-14T05:37:11.615632Z","iopub.status.idle":"2022-09-14T05:37:42.002496Z","shell.execute_reply.started":"2022-09-14T05:37:11.615589Z","shell.execute_reply":"2022-09-14T05:37:42.000869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\ndf_unique = pd.DataFrame(columns = ['features','% of unique values','% of NaNs'])\ndf_unique['features'] = train_dataset.columns.tolist()\nfor feature in tqdm.tqdm(train_dataset.columns):\n    df_unique.loc[df_unique.features == feature,'% of unique values'] = (len(train_dataset[feature].unique())/len(train_dataset[feature]))*100\n    df_unique.loc[df_unique.features == feature,'% of NaNs'] = train_dataset[feature].isnull().sum()/len(train_dataset[feature])*100\n# df_unique.sort_values(by = \"% of unique values\",ascending=False).head(40)\ndf_unique.sort_values(by = \"% of NaNs\",ascending=False).head(20) # Can we safely drop these features","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:37:42.004068Z","iopub.execute_input":"2022-09-14T05:37:42.004416Z","iopub.status.idle":"2022-09-14T05:38:08.535376Z","shell.execute_reply.started":"2022-09-14T05:37:42.004385Z","shell.execute_reply":"2022-09-14T05:38:08.534064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split columns in to numerical and categorical\ntrain_X = train_dataset.drop(\"target\", axis=1)\n# print(train_X.count())\n\nfeatures = train_X.columns.tolist()\nfeatures_num = train_X.select_dtypes(include=[np.number]).columns.to_list()\ncat_train_X = train_X.select_dtypes(include=['category'])\ndate_train_X = train_X.select_dtypes(include = ['datetime64'])\nfeatures_cat = cat_train_X.columns.to_list()\nprint(\"\\nCategorical features:\\n\", features_cat)","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:08.539647Z","iopub.execute_input":"2022-09-14T05:38:08.540514Z","iopub.status.idle":"2022-09-14T05:38:18.148347Z","shell.execute_reply.started":"2022-09-14T05:38:08.540458Z","shell.execute_reply":"2022-09-14T05:38:18.146967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"date_train_X.hist(xrot=45)\ndate_train_X.describe()","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:18.149967Z","iopub.execute_input":"2022-09-14T05:38:18.150432Z","iopub.status.idle":"2022-09-14T05:38:19.043166Z","shell.execute_reply.started":"2022-09-14T05:38:18.150396Z","shell.execute_reply":"2022-09-14T05:38:19.041633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# Explore in features that have little information\n\n# bad_features = []\n# for feature in tqdm.tqdm(features):\n#     if len(train_X[feature].unique()) == 1:\n#         bad_features.append(feature)\n# print(bad_features)\n# Fortunately there is no column with single value","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:19.050431Z","iopub.execute_input":"2022-09-14T05:38:19.053756Z","iopub.status.idle":"2022-09-14T05:38:19.058448Z","shell.execute_reply.started":"2022-09-14T05:38:19.053692Z","shell.execute_reply":"2022-09-14T05:38:19.057620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explore a bit more about the NaN entries\ncol_nan_index = train_X.isnull().any(axis=0) # columns that have nan\nnum_col_nan = train_X.isnull().any(axis=0).sum() # number of columns that have nan\nnum_data = train_X.shape[0]*train_X.shape[1] # number of entries\nnum_nan = train_X.isnull().values.sum() # number of nans\n\nprint(\"In 189 columns, {} columns that have NAs\".format(num_col_nan))\nprint(\"In train data, {:.2f} % are NAs\\n\".format(num_nan/num_data*100))\n\ncol_nan = col_nan_index[col_nan_index == True].index.tolist()\n\n# NAs = pd.concat([train_X.isnull().sum()], axis=1, keys=['Number of NAs'])\n# NAs = NAs[NAs.sum(axis=1) > 0].sort_values(by = \"Number of NAs\",ascending=False)\n# NAs[\"Percent\"] = NAs['Number of NAs'] / train_X.shape(0)\n# NAs[\"Percent\"] = NAs[\"Percent\"].mul(100).round(2) # astype(str).add(' %')\n# NAs[NAs[\"Percent\"]>90]","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:19.060458Z","iopub.execute_input":"2022-09-14T05:38:19.060884Z","iopub.status.idle":"2022-09-14T05:38:34.397701Z","shell.execute_reply.started":"2022-09-14T05:38:19.060851Z","shell.execute_reply":"2022-09-14T05:38:34.396149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Free RAM\ndel train_dataset\ndel cat_train_X\ndel date_train_X\ndel col_nan_index\ndel col_nan\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:34.399426Z","iopub.execute_input":"2022-09-14T05:38:34.399962Z","iopub.status.idle":"2022-09-14T05:38:34.641119Z","shell.execute_reply.started":"2022-09-14T05:38:34.399930Z","shell.execute_reply":"2022-09-14T05:38:34.639786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look inside those categorical columns\nfor feature in features_cat:\n    print(\"\\nUnique values of cat column_{}:\\n\".format(feature),train_X[feature].value_counts(dropna=False))\nprint(\"\\n===============Numerical columns below=================================\")\n# for feature in features_num[:5]:\n#     print(\"\\nUnique values of num column_{}:\\n\".format(feature),train_X[feature].value_counts(dropna=False))\n\n# It looks that the NaN or missing value in categorical column can be replaced with -1\n# And the data type should be changed to in8 to save space","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:34.643641Z","iopub.execute_input":"2022-09-14T05:38:34.644126Z","iopub.status.idle":"2022-09-14T05:38:35.268219Z","shell.execute_reply.started":"2022-09-14T05:38:34.644081Z","shell.execute_reply":"2022-09-14T05:38:35.266612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature engineering to get new features out of time series data","metadata":{}},{"cell_type":"markdown","source":"The data is injected with noise， according to the discovery in this notebook, so we will denoise it by using the same method.\nhttps://www.kaggle.com/code/raddar/amex-data-int-types-train/notebook\nA discussion has talked different normalization methods have beend applied to different columns of the data. Therefore, a standardization would not be necessary, especially cosidering tree models are robust to inputs.\nhttps://www.kaggle.com/competitions/amex-default-prediction/discussion/337920","metadata":{}},{"cell_type":"code","source":"# Let's make it right by replacing the \"nan\"s in the categorical data\n# features_CtoN = feature_cat[2:]\n\n# Converting blank entries in column \"D_64\" to numeric NaN\ntrain_X[\"D_64\"][train_X[\"D_64\"]==\"\"] = np.nan\n# train_X[\"D_63\"][train_X[\"D_63\"]==\"\"] = np.nan\n\n# for feature in features_CtoN:\n#     train_X[feature] = pd.factorize(train_X[feature])[0]\n    \n# check up\n# train_X.info()\n# feature_D_64_2 = np.unique(train_X['D_64'].values.tolist())\n# feature_D_63_2 = np.unique(train_X['D_63'].values.tolist())","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:35.269773Z","iopub.execute_input":"2022-09-14T05:38:35.270224Z","iopub.status.idle":"2022-09-14T05:38:35.291838Z","shell.execute_reply.started":"2022-09-14T05:38:35.270192Z","shell.execute_reply":"2022-09-14T05:38:35.290315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Encode categorical variables with Ordinal encoder\nfrom sklearn.preprocessing import OrdinalEncoder\nenc = OrdinalEncoder()\ntrain_X[features_cat] = enc.fit_transform(train_X[features_cat])\n\n# x['D_63'] = x['D_63'].apply(lambda t: {'CR':0, 'XZ':1, 'XM':2, 'CO':3, 'CL':4, 'XL':5}[t]).astype(np.int8)\n# x['D_64'] = x['D_64'].apply(lambda t: {np.nan:-1, 'O':0, '-1':1, 'R':2, 'U':3}[t]).astype(np.int8)\n\n# Replacing category NaN with -1 \n\nfor feature in tqdm.tqdm(features_cat):\n    train_X[feature] = train_X[feature].fillna(-1).astype(np.int8)","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:35.293529Z","iopub.execute_input":"2022-09-14T05:38:35.293869Z","iopub.status.idle":"2022-09-14T05:38:46.942533Z","shell.execute_reply.started":"2022-09-14T05:38:35.293839Z","shell.execute_reply":"2022-09-14T05:38:46.941100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for feature in features_cat:\n#     print(\"\\nUnique values of cat column_{}:\\n\".format(feature),train_X[feature].value_counts(dropna=False))","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:46.944379Z","iopub.execute_input":"2022-09-14T05:38:46.944843Z","iopub.status.idle":"2022-09-14T05:38:46.951330Z","shell.execute_reply.started":"2022-09-14T05:38:46.944799Z","shell.execute_reply":"2022-09-14T05:38:46.949732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Replace numerical NaN and denoise\n#### https://www.kaggle.com/code/raddar/amex-data-int-types-train/notebook","metadata":{"execution":{"iopub.status.busy":"2022-09-14T03:30:35.696864Z","iopub.execute_input":"2022-09-14T03:30:35.697202Z","iopub.status.idle":"2022-09-14T03:30:36.942178Z","shell.execute_reply.started":"2022-09-14T03:30:35.697168Z","shell.execute_reply":"2022-09-14T03:30:36.940939Z"}}},{"cell_type":"code","source":"def floorify(x, lo):\n    \"\"\"example: x in [0, 0.01] -> x := 0\"\"\"\n    return lo if x <= lo+0.01 and x >= lo else x\n\ndef floorify_zeros(x):\n    \"\"\"look around values [0,0.01] and determine if in proximity it's categorical. If yes - floorify\"\"\"\n    has_zeros = len([t for t in x if t>=0 and t<=0.01])>0 \n    no_proximity = len([t for t in x if t<0 and t>=-0.01])==0 and len([t for t in x if t>0.01 and t<=0.02])==0\n    if not no_proximity:\n        return x\n    if not has_zeros:\n        return x\n    x = [floorify(t, 0.0) for t in x]\n    return x\n\ndef floorify_ones(x):\n    \"\"\"look around values [1,1.01] and determine if in proximity it's categorical. If yes - floorify\"\"\"    \n    has_ones = len([t for t in x if t>=1 and t<=1.01])>0 \n    no_proximity = len([t for t in x if t<1 and t>=0.99])==0 and len([t for t in x if t>1.01 and t<=1.02])==0\n    if not no_proximity:\n        return x\n    if not has_ones:\n        return x\n    x = [floorify(t, 1.0) for t in x]\n    return x\n\ndef convert_na(x):\n    \"\"\"nan -> -1 if positive values\"\"\"\n    if np.nanmin(x)>=0:\n        return [-1 if np.isnan(t) else t for t in x]\n\ndef convert_to_int(x):\n    \"\"\"float -> int8 if possible\"\"\"\n    q = convert_na(x)\n    if set(np.unique(q)).union({-1,0,1}) == {-1,0,1}:\n        return [np.int8(t) for t in q]\n    return x\n\ndef floorify_ones_and_zeros(t):\n    \"\"\"do everything\"\"\"\n    t = floorify_zeros(t)\n    t = floorify_ones(t)\n    t = convert_to_int(t)\n    return t\n\ndef floorify_frac(x, interval=1):\n    \"\"\"convert to int if float appears ordinal\"\"\"\n    xt = (np.floor(x/interval+1e-6)).fillna(-1)\n    if np.max(xt)<=127:\n        return xt.astype(np.int8)\n    return xt.astype(np.int16)  ","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:46.953724Z","iopub.execute_input":"2022-09-14T05:38:46.954142Z","iopub.status.idle":"2022-09-14T05:38:46.971659Z","shell.execute_reply.started":"2022-09-14T05:38:46.954088Z","shell.execute_reply":"2022-09-14T05:38:46.970642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_data(train_X:pd.DataFrame):\n    # can be rounded up as each bin has AUC of 0.5\n    train_X['B_19'] = np.floor(train_X['B_19']*100).fillna(-1).astype(np.int8)\n    \n    train_X['B_4'] = floorify_frac(train_X['B_4'],1/78)\n    train_X['B_16'] = floorify_frac(train_X['B_16'],1/12)\n    train_X['B_20'] = floorify_frac(train_X['B_20'],1/17)\n    train_X['B_22'] = floorify_frac(train_X['B_22'],1/2)\n    train_X['B_30'] = floorify_frac(train_X['B_30'])\n    train_X['B_31'] = floorify_frac(train_X['B_31'])\n    train_X['B_32'] = floorify_frac(train_X['B_32'])\n    train_X['B_33'] = floorify_frac(train_X['B_33'])\n    train_X['B_38'] = floorify_frac(train_X['B_38'])\n    train_X['B_41'] = floorify_frac(train_X['B_41'])\n    train_X['D_39'] = floorify_frac(train_X['D_39'],1/34)\n    train_X['D_44'] = floorify_frac(train_X['D_44'],1/8)\n    train_X['D_49'] = floorify_frac(train_X['D_49'],1/71)\n    train_X['D_51'] = floorify_frac(train_X['D_51'],1/3)\n    train_X['D_59'] = floorify_frac(train_X['D_59']+5/48,1/48)\n    train_X['D_65'] = floorify_frac(train_X['D_65'],1/38)\n    train_X['D_66'] = floorify_frac(train_X['D_66'])\n    train_X['D_68'] = floorify_frac(train_X['D_68'])\n    train_X['D_70'] = floorify_frac(train_X['D_70'],1/4)\n    train_X['D_72'] = floorify_frac(train_X['D_72'],1/3)\n    train_X['D_74'] = floorify_frac(train_X['D_74'],1/14)\n    train_X['D_75'] = floorify_frac(train_X['D_75'],1/15)\n    train_X['D_78'] = floorify_frac(train_X['D_78'],1/2)\n    train_X['D_79'] = floorify_frac(train_X['D_79'],1/2)\n    train_X['D_80'] = floorify_frac(train_X['D_80'],1/5)\n    train_X['D_81'] = floorify_frac(train_X['D_81'])\n    train_X['D_82'] = floorify_frac(train_X['D_82'],1/2)\n    train_X['D_83'] = floorify_frac(train_X['D_83'])\n    train_X['D_84'] = floorify_frac(train_X['D_84'],1/2)\n    train_X['D_86'] = floorify_frac(train_X['D_86'])\n    train_X['D_87'] = floorify_frac(train_X['D_87'])\n    train_X['D_89'] = floorify_frac(train_X['D_89'],1/9)\n    train_X['D_92'] = floorify_frac(train_X['D_92'])\n    train_X['D_93'] = floorify_frac(train_X['D_93'])\n    train_X['D_94'] = floorify_frac(train_X['D_94'])\n    train_X['D_96'] = floorify_frac(train_X['D_96'])\n    train_X['D_103'] = floorify_frac(train_X['D_103'])\n    train_X['D_106'] = floorify_frac(train_X['D_106'],1/23)\n    train_X['D_107'] = floorify_frac(train_X['D_107'],1/3)\n    train_X['D_108'] = floorify_frac(train_X['D_108'])\n    train_X['D_109'] = floorify_frac(train_X['D_109'])\n    train_X['D_111'] = floorify_frac(train_X['D_111'],1/2)\n    train_X['D_113'] = floorify_frac(train_X['D_113'],1/5)\n    train_X['D_114'] = floorify_frac(train_X['D_114'])\n    train_X['D_116'] = floorify_frac(train_X['D_116'])\n    train_X['D_117'] = floorify_frac(train_X['D_117']+1)\n    train_X['D_120'] = floorify_frac(train_X['D_120'])\n    train_X['D_122'] = floorify_frac(train_X['D_122'],1/7)\n    train_X['D_123'] = floorify_frac(train_X['D_123'])\n    train_X['D_124'] = floorify_frac(train_X['D_124']+1/22,1/22)\n    train_X['D_125'] = floorify_frac(train_X['D_125'])\n    train_X['D_126'] = floorify_frac(train_X['D_126']+1)\n    train_X['D_127'] = floorify_frac(train_X['D_127'])\n    train_X['D_129'] = floorify_frac(train_X['D_129'])\n    train_X['D_135'] = floorify_frac(train_X['D_135'])\n    train_X['D_136'] = floorify_frac(train_X['D_136'],1/4)\n    train_X['D_137'] = floorify_frac(train_X['D_137'])\n    train_X['D_138'] = floorify_frac(train_X['D_138'],1/2)\n    train_X['D_139'] = floorify_frac(train_X['D_139'])\n    train_X['D_140'] = floorify_frac(train_X['D_140'])\n    train_X['D_143'] = floorify_frac(train_X['D_143'])\n    train_X['D_145'] = floorify_frac(train_X['D_145'],1/11)\n    train_X['R_2'] = floorify_frac(train_X['R_2'])\n    train_X['R_3'] = floorify_frac(train_X['R_3'],1/10)\n    train_X['R_4'] = floorify_frac(train_X['R_4'])\n    train_X['R_5'] = floorify_frac(train_X['R_5'],1/2)\n    train_X['R_8'] = floorify_frac(train_X['R_8'])\n    train_X['R_9'] = floorify_frac(train_X['R_9'],1/6)\n    train_X['R_10'] = floorify_frac(train_X['R_10'])\n    train_X['R_11'] = floorify_frac(train_X['R_11'],1/2)\n    train_X['R_13'] = floorify_frac(train_X['R_13'],1/31)\n    train_X['R_15'] = floorify_frac(train_X['R_15'])\n    train_X['R_16'] = floorify_frac(train_X['R_16'],1/2)\n    train_X['R_17'] = floorify_frac(train_X['R_17'],1/35)\n    train_X['R_18'] = floorify_frac(train_X['R_18'],1/31)\n    train_X['R_19'] = floorify_frac(train_X['R_19'])\n    train_X['R_20'] = floorify_frac(train_X['R_20'])\n    train_X['R_21'] = floorify_frac(train_X['R_21'])\n    train_X['R_22'] = floorify_frac(train_X['R_22'])\n    train_X['R_23'] = floorify_frac(train_X['R_23'])\n    train_X['R_24'] = floorify_frac(train_X['R_24'])\n    train_X['R_25'] = floorify_frac(train_X['R_25'])\n    train_X['R_26'] = floorify_frac(train_X['R_26'],1/28)\n    train_X['R_28'] = floorify_frac(train_X['R_28'])\n    train_X['S_6'] = floorify_frac(train_X['S_6'])\n    train_X['S_11'] = floorify_frac(train_X['S_11']+5/25,1/25)\n    train_X['S_15'] = floorify_frac(train_X['S_15']+3/10,1/10)\n    train_X['S_18'] = floorify_frac(train_X['S_18'])\n    train_X['S_20'] = floorify_frac(train_X['S_20'])\n\n    # one value overlaps, but the split can identified by S_11\n    train_X.loc[train_X.S_13.between(0.67, 0.7) & (train_X.S_11.isin([15,16,17])),'S_13'] = 0.6789168283158535\n    floor_vals = (0, 0.0377176456223467, 0.2804642206328049, 0.4013539714415651, 0.4206963381303189, 0.5067698438641042, \n                  0.5261121975338173, 0.5551258157960416, 0.6218568673028206, 0.6876208933830246, 0.8433269036807703, 1)\n    for c in floor_vals:\n        train_X['S_13'] = train_X['S_13'].apply(lambda t: floorify(t,c))\n    train_X['S_13'] = np.round(train_X['S_13']*1034).fillna(-1).astype(np.int16)  \n\n    # this one has many more value overlaps, but the splits can be identified by S_15\n    train_X.loc[(train_X.S_8>=0.30) & (train_X.S_8<=0.35) & (train_X.S_15<=6),'S_8'] = 0.3224889650033656\n    train_X.loc[(train_X.S_8>=0.30) & (train_X.S_8<=0.35) & (train_X.S_15==7),'S_8'] = 0.3145925513763017\n    train_X.loc[(train_X.S_8>=0.45) & (train_X.S_8<=0.477) & (train_X.S_15==3),'S_8'] = 0.4570436553944634\n    train_X.loc[(train_X.S_8>=0.45) & (train_X.S_8<=0.477) & (train_X.S_15==5),'S_8'] = 0.4636765662005172\n    train_X.loc[(train_X.S_8>=0.45) & (train_X.S_8<=0.477) & (train_X.S_15==6),'S_8'] = 0.4592546209653157\n    train_X.loc[(train_X.S_8>=0.55) & (train_X.S_8<=0.65) & (train_X.S_15==5),'S_8'] = 0.5938092592144236\n    train_X.loc[(train_X.S_8>=0.55) & (train_X.S_8<=0.65) & (train_X.S_15==4),'S_8'] = 0.5994946974629933\n    train_X.loc[(train_X.S_8>=0.55) & (train_X.S_8<=0.65) & (train_X.S_15<=2),'S_8'] = 0.6017056828901041\n    train_X.loc[(train_X.S_8>=0.73) & (train_X.S_8<=0.78) & (train_X.S_15==3),'S_8'] = 0.7441567340107059\n    train_X.loc[(train_X.S_8>=0.73) & (train_X.S_8<=0.78) & (train_X.S_15==5),'S_8'] = 0.7517372106519937\n    train_X.loc[(train_X.S_8>=0.73) & (train_X.S_8<=0.78) & (train_X.S_15==4),'S_8'] = 0.7586861099807893\n    train_X.loc[(train_X.S_8>=0.91) & (train_X.S_8<=0.98) & (train_X.S_15==4),'S_8'] = 0.9147189165383852\n    train_X.loc[(train_X.S_8>=0.91) & (train_X.S_8<=0.98) & (train_X.S_15<=2),'S_8'] = 0.9327230426634736\n    train_X.loc[(train_X.S_8>=0.91) & (train_X.S_8<=0.98) & (train_X.S_15==3),'S_8'] = 0.935565546481781\n    train_X.loc[(train_X.S_8>=1.12) & (train_X.S_8<=1.17) & (train_X.S_15<=2),'S_8'] = 1.1440303975988897\n    train_X.loc[(train_X.S_8>=1.12) & (train_X.S_8<=1.17) & (train_X.S_15==3),'S_8'] = 1.151926881019957\n    floor_vals = (0, 0.1017056275625063, 0.119709415455368, 0.1667719530078215, 0.2438408100936861, \n                  0.3578648754166172, 0.4055590769093041, 0.4772583808904347, 0.4876816287061991, \n                  0.6620341135675392, 0.7005685574395781, 0.8509160456526623, 1, 1.0145299163657109, \n                  1.1051803467580654, 1.2214158871037435)\n    for c in tqdm.tqdm(floor_vals):    \n        train_X['S_8'] = train_X['S_8'].apply(lambda t: floorify(t,c))\n    \n    train_X['S_8'] = np.round(train_X['S_8']*3166).fillna(-1).astype(np.int16)\n\n    cols = train_X.select_dtypes(include=[float]).columns\n    for col in tqdm.tqdm(cols):\n        train_X[col] = floorify_ones_and_zeros(train_X[col])\n\n    for col in train_X.select_dtypes(include=[float]).columns.tolist():\n        train_X[col] = train_X[col].astype(np.float32)\n    return train_X","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:46.973701Z","iopub.execute_input":"2022-09-14T05:38:46.974441Z","iopub.status.idle":"2022-09-14T05:38:47.025797Z","shell.execute_reply.started":"2022-09-14T05:38:46.974405Z","shell.execute_reply":"2022-09-14T05:38:47.024658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = process_data(train_X)","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:38:47.027552Z","iopub.execute_input":"2022-09-14T05:38:47.027951Z","iopub.status.idle":"2022-09-14T05:41:38.262923Z","shell.execute_reply.started":"2022-09-14T05:38:47.027917Z","shell.execute_reply":"2022-09-14T05:41:38.261717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explore a bit more about the NaN entries\ncol_nan_index = train_X.isnull().any(axis=0) # columns that have nan\ncol_nan = col_nan_index[col_nan_index == True].index.tolist()\nnum_col_nan = train_X.isnull().any(axis=0).sum() # number of columns that have nan\n# num_data = train_X.shape[0]*train_X.shape[1] # number of entries\n# num_nan = train_X.isnull().values.sum() # number of nans\n\nprint(\"In 189 columns, {} columns that have NAs\".format(num_col_nan))\n# print(\"In train data, {:.2f} % are NAs\\n\".format(num_nan/num_data*100))\n# for col in col_nan[:3]:\n#     print(\"\\nUnique values of num column_{}:\\n\".format(col),train_X[col].value_counts(dropna=False))\ntrain_X.info()","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:41:38.265068Z","iopub.execute_input":"2022-09-14T05:41:38.265458Z","iopub.status.idle":"2022-09-14T05:41:43.288359Z","shell.execute_reply.started":"2022-09-14T05:41:38.265423Z","shell.execute_reply":"2022-09-14T05:41:43.286758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:41:43.290716Z","iopub.execute_input":"2022-09-14T05:41:43.291190Z","iopub.status.idle":"2022-09-14T05:41:43.450251Z","shell.execute_reply.started":"2022-09-14T05:41:43.291145Z","shell.execute_reply":"2022-09-14T05:41:43.448671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# turn the rest NA to 0\nfor col in tqdm.tqdm(col_nan):\n    train_X[col] = train_X[col].fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:41:43.452112Z","iopub.execute_input":"2022-09-14T05:41:43.453209Z","iopub.status.idle":"2022-09-14T05:41:46.049372Z","shell.execute_reply.started":"2022-09-14T05:41:43.453169Z","shell.execute_reply":"2022-09-14T05:41:46.047982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Encode categorical variables with one hot encoding\n\n# for feature in features_cat:\n#     train_X2 = pd.get_dummies(train_X2, columns=[feature])","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:41:46.054679Z","iopub.execute_input":"2022-09-14T05:41:46.055114Z","iopub.status.idle":"2022-09-14T05:41:46.060028Z","shell.execute_reply.started":"2022-09-14T05:41:46.055075Z","shell.execute_reply":"2022-09-14T05:41:46.058882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the train_X for further use\n\nprint(\"\\n The shape of train data is:\", train_X.shape)\n\n# os.mkdir('../data2')\ntrain_X.reset_index(drop=True,inplace=True)\n\n# train_X = train_X.drop(columns=[\"level_0\",\"index\"],axis=1)\ntrain_X.sample(3)\n# X.sample(3)\n\ntrain_X.to_feather(\"./train_X.feather\")\n\n# del train_X\ndel train_X\ndel col_nan\ndel col_nan_index\ndel enc\ndel features\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:41:46.061707Z","iopub.execute_input":"2022-09-14T05:41:46.062077Z","iopub.status.idle":"2022-09-14T05:41:51.405782Z","shell.execute_reply.started":"2022-09-14T05:41:46.062043Z","shell.execute_reply":"2022-09-14T05:41:51.404548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_dataset = train_dataset_.groupby('customer_ID').tail(1).set_index('customer_ID', drop=True).sort_index()\n# print(train_dataset.shape)\n\n# del train_df_pd\n# gc.collect()\n# del train_dataset_\n# gc.collect()\n# train_dataset.head()\n# train_dataset.info()","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:41:51.407540Z","iopub.execute_input":"2022-09-14T05:41:51.408333Z","iopub.status.idle":"2022-09-14T05:41:51.414929Z","shell.execute_reply.started":"2022-09-14T05:41:51.408287Z","shell.execute_reply":"2022-09-14T05:41:51.413022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Normalize data\n# from sklearn.preprocessing import StandardScaler\n# scaler = StandardScaler()\n# train_X[features_num] = scaler.fit_transform(train_X[features_num])","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:41:51.417071Z","iopub.execute_input":"2022-09-14T05:41:51.417667Z","iopub.status.idle":"2022-09-14T05:41:51.435193Z","shell.execute_reply.started":"2022-09-14T05:41:51.417623Z","shell.execute_reply":"2022-09-14T05:41:51.433654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# Apply the same to test data\ntest_data = pd.read_feather('../input/amexfeather/test_data.ftr')\ntest_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:41:51.437190Z","iopub.execute_input":"2022-09-14T05:41:51.437704Z","iopub.status.idle":"2022-09-14T05:42:31.539933Z","shell.execute_reply.started":"2022-09-14T05:41:51.437660Z","shell.execute_reply":"2022-09-14T05:42:31.538731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_unique2 = pd.DataFrame(columns = ['features','% of unique values','% of NaNs'])\ndf_unique2['features'] = test_data.columns.tolist()\nfor feature in tqdm.tqdm(test_data.columns):\n    df_unique2.loc[df_unique2.features == feature,'% of unique values'] = (len(test_data[feature].unique())/len(test_data[feature]))*100\n    df_unique2.loc[df_unique2.features == feature,'% of NaNs'] = test_data[feature].isnull().sum()/len(test_data[feature])*100\n# df_unique.sort_values(by = \"% of unique values\",ascending=False).head(40)\ndf_unique2.sort_values(by = \"% of NaNs\",ascending=False).head(20) # Can we safely drop these features","metadata":{"execution":{"iopub.status.busy":"2022-09-14T05:42:31.541821Z","iopub.execute_input":"2022-09-14T05:42:31.542311Z","iopub.status.idle":"2022-09-14T05:43:20.026789Z","shell.execute_reply.started":"2022-09-14T05:42:31.542264Z","shell.execute_reply":"2022-09-14T05:43:20.025039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_unique3 = df_unique.copy()\ndf_unique3 = df_unique3.merge(df_unique2, how='inner', on=\"features\")\ndf_unique3 = df_unique3.sort_values(by = \"% of NaNs_x\",ascending=False)\ndf_unique3.to_csv(\"./NaN_percent.csv\")\n\n# del train_X\ndel df_unique\ndel df_unique2\ndel df_unique3\n\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# features = test_data.columns.tolist()\n# features_num = test_data.select_dtypes(include=[np.number]).columns.to_list()\n# features_cat = test_data.select_dtypes(include=['category']).columns.to_list()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explore a bit more about the NaN entries\ncol_nan_index2 = test_data.isnull().any(axis=0) # columns that have nan\nnum_col_nan2 = test_data.isnull().any(axis=0).sum() # number of columns that have nan\n# num_data2 = test_data.shape[0]*test_X.shape[1] # number of entries\n# num_nan2 = test_data.isnull().values.sum() # number of nans\n\nprint(\"In 189 columns, {} columns that have NAs\".format(num_col_nan2))\n# print(\"In test data, {:.2f} % are NAs\\n\".format(num_nan2/num_data2*100))\n\ncol_nan2 = col_nan_index2[col_nan_index2 == True].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-09-14T04:45:10.239916Z","iopub.execute_input":"2022-09-14T04:45:10.240391Z","iopub.status.idle":"2022-09-14T04:45:32.622128Z","shell.execute_reply.started":"2022-09-14T04:45:10.240351Z","shell.execute_reply":"2022-09-14T04:45:32.620653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data[\"D_64\"][test_data[\"D_64\"]==\"\"] = np.nan\n\n# Encode categorical variables with Ordinal encoder\nenc = OrdinalEncoder()\ntest_data[features_cat] = enc.fit_transform(test_data[features_cat])\n\n# Replacing category NaN with -1 \nfor feature in tqdm.tqdm(features_cat):\n    test_data[feature] = test_data[feature].fillna(-1).astype(np.int8)","metadata":{"execution":{"iopub.status.busy":"2022-09-14T04:47:44.267771Z","iopub.execute_input":"2022-09-14T04:47:44.268267Z","iopub.status.idle":"2022-09-14T04:48:10.500585Z","shell.execute_reply.started":"2022-09-14T04:47:44.268231Z","shell.execute_reply":"2022-09-14T04:48:10.499245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del enc\ndel col_nan2\ndel col_nan_index2\ndel features_num\ndel features_cat\n\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = process_data(test_data)","metadata":{"execution":{"iopub.status.busy":"2022-09-14T04:49:31.974442Z","iopub.execute_input":"2022-09-14T04:49:31.975677Z","iopub.status.idle":"2022-09-14T04:55:16.651332Z","shell.execute_reply.started":"2022-09-14T04:49:31.975617Z","shell.execute_reply":"2022-09-14T04:55:16.650293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Explore a bit more about the NaN entries\ncol_nan_index2 = test_data.isnull().any(axis=0) # columns that have nan\nnum_col_nan2 = test_data.isnull().any(axis=0).sum() # number of columns that have nan\n\nprint(\"In 189 columns, {} columns that have NAs\".format(num_col_nan2))\ncol_nan2 = col_nan_index2[col_nan_index2 == True].index.tolist()\n\n# turn the rest NA to 0\nfor col in tqdm.tqdm(col_nan2):\n    test_data[col] = test_data[col].fillna(0)\n\n# check\ntest_data.isnull().any(axis=0).sum()","metadata":{"execution":{"iopub.status.busy":"2022-09-14T04:56:30.428848Z","iopub.execute_input":"2022-09-14T04:56:30.429311Z","iopub.status.idle":"2022-09-14T04:56:52.697084Z","shell.execute_reply.started":"2022-09-14T04:56:30.429278Z","shell.execute_reply":"2022-09-14T04:56:52.695497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save for further use\n\nprint(\"\\n The shape of test data is:\", test_data.shape)\n\n# os.mkdir('../data2')\ntest_data.reset_index(drop=True,inplace=True)\n\n# test_data = test_data.drop(columns=[\"level_0\",\"index\"],axis=1)\ntest_data.sample(3)\n# X.sample(3)\n\ntest_data.to_feather(\"./test_X.feather\")\n\n# del train_X\ndel test_data\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}