{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Categorical variables\nWe notice that categoricals features are not the same over time for a customer.\nFor example:\n- D_63 changes over time with: 'CR', 'CR', 'CR', 'CR', 'CR', 'XM', 'XZ', 'CO', 'CL', 'XL', 'XL','XZ'\n- D_64 changes over time with: 'R', 'U', 'O'\n\nAt some point capture of categorical is not possible even if it was good in the past.\n- D_64 changes over time with: 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', nan, nan, nan, 'O'\n\nWe can encode target for categories per statement, rank=0 <=> 'last', rank=12 <=> first (if any).","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport random, os\nfrom sklearn.model_selection import StratifiedKFold\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\npd.set_option('display.max_columns', 50)\npd.set_option('display.max_rows', 2000) \npd.set_option('display.max_colwidth', None)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T16:50:34.774185Z","iopub.execute_input":"2022-07-17T16:50:34.775164Z","iopub.status.idle":"2022-07-17T16:50:34.782802Z","shell.execute_reply.started":"2022-07-17T16:50:34.775132Z","shell.execute_reply":"2022-07-17T16:50:34.781221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(s):\n    random.seed(s)\n    os.environ['PYTHONHASHSEED'] = str(s)\n    np.random.seed(s)\n    \nseed = 42\n\nseed_everything(seed)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T16:10:52.188763Z","iopub.execute_input":"2022-07-17T16:10:52.189674Z","iopub.status.idle":"2022-07-17T16:10:52.197238Z","shell.execute_reply.started":"2022-07-17T16:10:52.189600Z","shell.execute_reply":"2022-07-17T16:10:52.195730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CATEGORICALS = sorted(['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68'])\nTIME = \"S_2\"\nCID = \"customer_ID\"\nDAY_OF_WEEK = \"day_of_week\"\nTARGET = \"target\"","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:41:02.193400Z","iopub.execute_input":"2022-07-17T15:41:02.194150Z","iopub.status.idle":"2022-07-17T15:41:02.198350Z","shell.execute_reply.started":"2022-07-17T15:41:02.194121Z","shell.execute_reply":"2022-07-17T15:41:02.197658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_pd = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_data.csv\", usecols=[CID, TIME] + CATEGORICALS + [\"P_2\"])\ncat_pd[TIME] = pd.to_datetime(cat_pd[TIME])\ncat_pd[DAY_OF_WEEK] = cat_pd[TIME].dt.dayofweek\nlabels_pd = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\")\ncat_pd = pd.merge(cat_pd, labels_pd, on=CID)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:30:27.691970Z","iopub.execute_input":"2022-07-17T15:30:27.692832Z","iopub.status.idle":"2022-07-17T15:35:40.723724Z","shell.execute_reply.started":"2022-07-17T15:30:27.692788Z","shell.execute_reply":"2022-07-17T15:35:40.721938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preview\ncat_pd.tail(16)","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:35:40.726936Z","iopub.execute_input":"2022-07-17T15:35:40.727391Z","iopub.status.idle":"2022-07-17T15:35:40.800040Z","shell.execute_reply.started":"2022-07-17T15:35:40.727338Z","shell.execute_reply":"2022-07-17T15:35:40.797445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Basic describe\nfor c in CATEGORICALS:\n    print(c, cat_pd[c].nunique(), cat_pd[c].unique())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:35:40.801854Z","iopub.execute_input":"2022-07-17T15:35:40.802157Z","iopub.status.idle":"2022-07-17T15:35:43.876898Z","shell.execute_reply.started":"2022-07-17T15:35:40.802132Z","shell.execute_reply":"2022-07-17T15:35:43.875634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Is category the same over time for a customer?","metadata":{}},{"cell_type":"code","source":"for c in CATEGORICALS:\n    tmp_cat = cat_pd.groupby([CID])[c].agg(['nunique', 'unique']).reset_index()\n    tmp_cat = tmp_cat.sort_values(['nunique'], ascending=[False])\n    print(c, tmp_cat[\"nunique\"].min(), tmp_cat[\"nunique\"].max())\n    display(tmp_cat.head())","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:35:43.879878Z","iopub.execute_input":"2022-07-17T15:35:43.881188Z","iopub.status.idle":"2022-07-17T15:40:26.634195Z","shell.execute_reply.started":"2022-07-17T15:35:43.881130Z","shell.execute_reply":"2022-07-17T15:40:26.631994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No, it's not","metadata":{}},{"cell_type":"code","source":"UIDS = [\"a164c21439437ed3b608cdb90d3d0405ed49bb2279bed286b708d5e114cec7da\", \"0f3a7ffb91f95bb44f90c4f25cb8a719bee43e18ff69048f9be7df5e6399e599\", \n        \"e76b230bdbe5fc136bd4bd573d087f033c083e182a3ce4ddf404199223816508\", \"ec4257b841e6ef0b0eaa8e6c638394e91860f6ca6c0ae56eb592f26dd8b78a59\"]","metadata":{"execution":{"iopub.status.busy":"2022-07-17T16:38:39.404781Z","iopub.execute_input":"2022-07-17T16:38:39.405151Z","iopub.status.idle":"2022-07-17T16:38:39.410215Z","shell.execute_reply.started":"2022-07-17T16:38:39.405123Z","shell.execute_reply":"2022-07-17T16:38:39.409427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for uid in UIDS:\n    display(cat_pd[cat_pd[CID] == uid])","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:46:32.347623Z","iopub.execute_input":"2022-07-17T15:46:32.347978Z","iopub.status.idle":"2022-07-17T15:46:34.255933Z","shell.execute_reply.started":"2022-07-17T15:46:32.347953Z","shell.execute_reply":"2022-07-17T15:46:34.254552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Target mean per category","metadata":{}},{"cell_type":"code","source":"for c in CATEGORICALS:\n    tmp_cat = cat_pd.groupby([c])[TARGET].mean().reset_index()\n    d = sns.barplot(data=tmp_cat, x=c, y=TARGET)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:42:36.897481Z","iopub.execute_input":"2022-07-17T15:42:36.897930Z","iopub.status.idle":"2022-07-17T15:42:40.527302Z","shell.execute_reply.started":"2022-07-17T15:42:36.897900Z","shell.execute_reply":"2022-07-17T15:42:40.526137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Target encoding of the each statement / month\n- 0 = last (most recent)\n- 12 = first (if any)","metadata":{}},{"cell_type":"code","source":"def pivot_data(df, train=True):\n    cols = [c for c in df.columns if c not in [CID, TIME, TARGET]]\n    tmp = df.copy()\n    tmp['max'] = tmp.groupby([CID])[TIME].transform('max')\n    tmp['size'] = tmp.groupby([CID])[TIME].transform('size')\n    tmp['rank'] = tmp.groupby([CID])[TIME].transform('rank')\n    tmp['statement'] = (tmp['size']-tmp['rank']).astype(np.int8)\n    pivot_pd = tmp.pivot(index=CID,columns=['statement'],values=cols)\n    pivot_pd.columns = [('{0}__{1}'.format(*tup)) for tup in pivot_pd.columns]\n    pivot_pd = pivot_pd.reset_index()\n    return pivot_pd","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:43:00.895912Z","iopub.execute_input":"2022-07-17T15:43:00.896425Z","iopub.status.idle":"2022-07-17T15:43:00.907709Z","shell.execute_reply.started":"2022-07-17T15:43:00.896390Z","shell.execute_reply":"2022-07-17T15:43:00.906859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CAT_COL = \"B_30\" # DAY_OF_WEEK\nte_col = \"%s_TE\" % CAT_COL","metadata":{"execution":{"iopub.status.busy":"2022-07-17T15:43:46.614665Z","iopub.execute_input":"2022-07-17T15:43:46.615081Z","iopub.status.idle":"2022-07-17T15:43:46.621314Z","shell.execute_reply.started":"2022-07-17T15:43:46.615049Z","shell.execute_reply":"2022-07-17T15:43:46.620584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Simple aggregated features\ntrain_pd = cat_pd.groupby(CID)[[\"P_2\"]].agg(['mean', 'std', 'min', 'max', 'last'])\nnum_cols =  ['_'.join(x) for x in train_pd.columns]\ntrain_pd.columns = num_cols\ntrain_pd = train_pd.reset_index()\ntrain_pd = pd.merge(train_pd, labels_pd, on=CID)\n\n# Define fold based on stratified target\nFOLDS = [1,2,3,4,5]\nkf = StratifiedKFold(n_splits = len(FOLDS), shuffle = True, random_state = seed)\nfor fold, (train_idx, valid_idx) in enumerate(kf.split(train_pd, train_pd[TARGET]), 1):\n    train_pd.loc[valid_idx, 'fold'] = fold\n\ntrain_pd[\"fold\"] = train_pd[\"fold\"].astype(np.int8)\ntrain_pd","metadata":{"execution":{"iopub.status.busy":"2022-07-17T16:13:00.028875Z","iopub.execute_input":"2022-07-17T16:13:00.030425Z","iopub.status.idle":"2022-07-17T16:13:02.953850Z","shell.execute_reply.started":"2022-07-17T16:13:00.030357Z","shell.execute_reply":"2022-07-17T16:13:02.952686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://medium.com/@pouryaayria/k-fold-target-encoding-dfe9a594874b\n# Encode target on train and use the same on validation","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_cols = [c for c in train_pd.columns if c not in [CID, TARGET, \"fold\"]]","metadata":{"execution":{"iopub.status.busy":"2022-07-17T16:36:46.706924Z","iopub.execute_input":"2022-07-17T16:36:46.707496Z","iopub.status.idle":"2022-07-17T16:36:46.715395Z","shell.execute_reply.started":"2022-07-17T16:36:46.707459Z","shell.execute_reply":"2022-07-17T16:36:46.714045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fold in FOLDS:\n    \n    print(\"Fold\", fold)\n    \n    # Train/Valid\n    X_train = train_pd[train_pd['fold'] != fold]\n    X_valid = train_pd[train_pd['fold'] == fold]\n    X_valid_cid = X_valid[CID].unique()\n        \n    # Target encoding on 2,3,4,5 (train)\n    X_train_cat_pd = cat_pd[~cat_pd[CID].isin(X_valid_cid)]\n    te_fold = X_train_cat_pd.groupby([CAT_COL])[TARGET].mean().to_dict()\n    X_train_cat_pd[te_col] = X_train_cat_pd[CAT_COL].map(te_fold)\n    # One column per month\n    X_train_pivot = pivot_data(X_train_cat_pd[[CID, TIME, te_col]])\n\n    # Apply on fold 1 (valid)\n    X_valid_cat_pd = cat_pd[cat_pd[CID].isin(X_valid_cid)]\n    X_valid_cat_pd[te_col] = X_valid_cat_pd[CAT_COL].map(te_fold)\n    # One column per month\n    X_valid_pivot = pivot_data(X_valid_cat_pd[[CID, TIME, te_col]])\n    \n    # Join target encoded columns to features\n    X_train = pd.merge(X_train, X_train_pivot, on=CID)\n    X_valid = pd.merge(X_valid, X_valid_pivot, on=CID)\n    \n    # Display sample of train/valid\n    display(X_train[X_train[CID].isin(UIDS)])    \n    display(X_valid[X_valid[CID].isin(UIDS)])\n    \n    print()","metadata":{"execution":{"iopub.status.busy":"2022-07-17T16:47:06.741742Z","iopub.execute_input":"2022-07-17T16:47:06.742169Z","iopub.status.idle":"2022-07-17T16:48:12.614870Z","shell.execute_reply.started":"2022-07-17T16:47:06.742141Z","shell.execute_reply":"2022-07-17T16:48:12.613531Z"},"trusted":true},"execution_count":null,"outputs":[]}]}