{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-25T21:00:08.469362Z","iopub.execute_input":"2022-07-25T21:00:08.473135Z","iopub.status.idle":"2022-07-25T21:00:08.550843Z","shell.execute_reply.started":"2022-07-25T21:00:08.472786Z","shell.execute_reply":"2022-07-25T21:00:08.548398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/rohanrao/amex-competition-metric-implementations\n# https://www.kaggle.com/competitions/amex-default-prediction/discussion/328020\nimport numpy as np\n\ndef amex_metric_numpy(y_true: np.array, y_pred: np.array) -> float:\n\n    # count of positives and negatives\n    n_pos = y_true.sum()\n    n_neg = y_true.shape[0] - n_pos\n\n    # sorting by descring prediction values\n    indices = np.argsort(y_pred)[::-1]\n    preds, target = y_pred[indices], y_true[indices]\n\n    # filter the top 4% by cumulative row weights\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_filter = cum_norm_weight <= 0.04\n\n    # default rate captured at 4%\n    d = target[four_pct_filter].sum() / n_pos\n\n    # weighted gini coefficient\n    lorentz = (target / n_pos).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    # max weighted gini coefficient\n    gini_max = 10 * n_neg * (1 - 19 / (n_pos + 20 * n_neg))\n\n    # normalized weighted gini coefficient\n    g = gini / gini_max\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:00:10.784526Z","iopub.execute_input":"2022-07-25T21:00:10.785776Z","iopub.status.idle":"2022-07-25T21:00:10.798326Z","shell.execute_reply.started":"2022-07-25T21:00:10.785722Z","shell.execute_reply":"2022-07-25T21:00:10.796521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:00:11.861127Z","iopub.execute_input":"2022-07-25T21:00:11.861629Z","iopub.status.idle":"2022-07-25T21:00:11.868154Z","shell.execute_reply.started":"2022-07-25T21:00:11.861587Z","shell.execute_reply":"2022-07-25T21:00:11.866843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The 11 columns ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68'] are categorical with maximum 8 values. Therefore each of these columns can be converted into int8 which is 1 byte per row. Originally they are each 8 bytes per row.\n\nall features should be kept, except those that negatively affect the result.\n\nremove all features those are highly correlated with the others\n\nD_63_last 0~5 they are ['CR', 'XZ', 'XM', 'CO', 'CL', 'XL']D_63_last\ttarget_mean\n0\t0.175285\n1\t0.229850\n2\t0.298736\n3\t0.270823\n4\t0.312863\n5\t0.449257\n\nfeature engineering where you aggregate the values of a specific customer_ID and get the mean ,std,min and max(so on and so forth)\n\nCheck groupby in-built functions for feature engineering","metadata":{}},{"cell_type":"markdown","source":"Autoencoder Notebooks\n* [Denoising Autoencoder](https://www.kaggle.com/code/osciiart/denoising-autoencoder)\n* [Supervised Emphasized Denoising AutoEncoder](https://www.kaggle.com/code/jeongyoonlee/supervised-emphasized-denoising-autoencoder)","metadata":{}},{"cell_type":"code","source":"# float to int conversion were done to remove noise (like with Balance variables)\nnew_train = pd.read_parquet(\"/kaggle/input/amex-data-integer-dtypes-parquet-format/train.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:00:15.485656Z","iopub.execute_input":"2022-07-25T21:00:15.486159Z","iopub.status.idle":"2022-07-25T21:00:36.603480Z","shell.execute_reply.started":"2022-07-25T21:00:15.486117Z","shell.execute_reply":"2022-07-25T21:00:36.602315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_train.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:00:36.606021Z","iopub.execute_input":"2022-07-25T21:00:36.606865Z","iopub.status.idle":"2022-07-25T21:00:36.649058Z","shell.execute_reply.started":"2022-07-25T21:00:36.606818Z","shell.execute_reply":"2022-07-25T21:00:36.647888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = new_train","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:00:36.652308Z","iopub.execute_input":"2022-07-25T21:00:36.653565Z","iopub.status.idle":"2022-07-25T21:00:36.657926Z","shell.execute_reply.started":"2022-07-25T21:00:36.653522Z","shell.execute_reply":"2022-07-25T21:00:36.656839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del new_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:00:36.660857Z","iopub.execute_input":"2022-07-25T21:00:36.661654Z","iopub.status.idle":"2022-07-25T21:00:36.961863Z","shell.execute_reply.started":"2022-07-25T21:00:36.661608Z","shell.execute_reply":"2022-07-25T21:00:36.960457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/amex-default-prediction/discussion/327143\n# train_df = pd.read_feather('../input/parquet-files-amexdefault-prediction/train_data.ftr')","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:17:06.591832Z","iopub.execute_input":"2022-07-24T03:17:06.592323Z","iopub.status.idle":"2022-07-24T03:17:06.613435Z","shell.execute_reply.started":"2022-07-24T03:17:06.592278Z","shell.execute_reply":"2022-07-24T03:17:06.612619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['customer_ID'] =\\\n    train_df['customer_ID'].apply(lambda x: int(x[-16:],16) ).astype('int64')\ntrain_df['S_2'] = pd.to_datetime(train_df['S_2'])","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:00:36.964995Z","iopub.execute_input":"2022-07-25T21:00:36.966054Z","iopub.status.idle":"2022-07-25T21:00:45.309625Z","shell.execute_reply.started":"2022-07-25T21:00:36.966012Z","shell.execute_reply":"2022-07-25T21:00:45.308216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_sample = train_df.iloc[0:10000, :]\ndel train_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:00:45.311405Z","iopub.execute_input":"2022-07-25T21:00:45.311842Z","iopub.status.idle":"2022-07-25T21:00:45.424814Z","shell.execute_reply.started":"2022-07-25T21:00:45.311794Z","shell.execute_reply":"2022-07-25T21:00:45.423659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_sample.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:00:45.426395Z","iopub.execute_input":"2022-07-25T21:00:45.427418Z","iopub.status.idle":"2022-07-25T21:00:45.456896Z","shell.execute_reply.started":"2022-07-25T21:00:45.427371Z","shell.execute_reply":"2022-07-25T21:00:45.455851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a new column which will contain SES calculation (for that customer_ID)\n# - you will likely have intermediate work columns\n# keep in mind the code should run relatively quickly\n\n# copy one column for a dataframe\nmy_work = train_df_sample[['customer_ID', 'S_2', 'B_1']].copy()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:03:52.992203Z","iopub.execute_input":"2022-07-25T21:03:52.992654Z","iopub.status.idle":"2022-07-25T21:03:53.000083Z","shell.execute_reply.started":"2022-07-25T21:03:52.992618Z","shell.execute_reply":"2022-07-25T21:03:52.999034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_example = my_work.head(13)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:28:06.074258Z","iopub.execute_input":"2022-07-25T21:28:06.074708Z","iopub.status.idle":"2022-07-25T21:28:06.082391Z","shell.execute_reply.started":"2022-07-25T21:28:06.074673Z","shell.execute_reply":"2022-07-25T21:28:06.080578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_example['B_1_ses_calc'] = ses_cal(small_example['B_1'])","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:42:32.558815Z","iopub.execute_input":"2022-07-25T21:42:32.559362Z","iopub.status.idle":"2022-07-25T21:42:32.570490Z","shell.execute_reply.started":"2022-07-25T21:42:32.559320Z","shell.execute_reply":"2022-07-25T21:42:32.568925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"small_example","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:42:41.717243Z","iopub.execute_input":"2022-07-25T21:42:41.717744Z","iopub.status.idle":"2022-07-25T21:42:41.737218Z","shell.execute_reply.started":"2022-07-25T21:42:41.717704Z","shell.execute_reply":"2022-07-25T21:42:41.735654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# simple exponential smoothing calculation\n# given a df, return a series\n# inspired by: https://otexts.com/fpp2/ses.html\n# Explore different functions of groupby: https://pandas.pydata.org/docs/reference/api/pandas.core.groupby.DataFrameGroupBy.transform.html\ndef ses_cal(series):\n    # closer to one means give even more weight to recent observations\n    alpha = 0.5\n    one_minus_alpha = 1 - alpha\n    \n#     # instantiate new colum\n#     series['ses_col'] = 0\n    \n    # you will likely have intermediate work columns\n    # consider iterating from the bottom up\n    num_statements = len(series)\n    \n    # the list of SES calculations to be returned\n    new_list_of_vals = []\n    \n    # calculating a val for each statement date\n    for i in range(num_statements-1, -1,-1):\n        tmp_val = 0\n        \n        if i == 0:\n            tmp_val = series[i]\n            new_list_of_vals.append(tmp_val)\n            break\n            \n        # get the val using this statement and previous statement dates\n        for j in range(i,-1,-1):\n            tmp_val += (alpha*series[j]*np.power(one_minus_alpha, i-j))\n        new_list_of_vals.append(tmp_val)\n    return new_list_of_vals[::-1]","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:38:18.609097Z","iopub.execute_input":"2022-07-25T21:38:18.609808Z","iopub.status.idle":"2022-07-25T21:38:18.621160Z","shell.execute_reply.started":"2022-07-25T21:38:18.609757Z","shell.execute_reply":"2022-07-25T21:38:18.619533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[columns_with_missing_vals] = train.groupby(\"customer_ID\")[columns_with_missing_vals].transform(lambda x: x.fillna(method='ffill'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# float to int conversion were done to remove noise (like with Balance variables)\nnew_test = pd.read_parquet(\"/kaggle/input/amex-data-integer-dtypes-parquet-format/test.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:33:04.658637Z","iopub.execute_input":"2022-07-24T21:33:04.659030Z","iopub.status.idle":"2022-07-24T21:33:42.067870Z","shell.execute_reply.started":"2022-07-24T21:33:04.658985Z","shell.execute_reply":"2022-07-24T21:33:42.067106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = new_test","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:33:42.070884Z","iopub.execute_input":"2022-07-24T21:33:42.071664Z","iopub.status.idle":"2022-07-24T21:33:42.080701Z","shell.execute_reply.started":"2022-07-24T21:33:42.071631Z","shell.execute_reply":"2022-07-24T21:33:42.078315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df = pd.read_feather('../input/parquet-files-amexdefault-prediction/test_data.ftr')\ntest_df_sample = test_df.iloc[0:10000, :]\ndel new_test\ndel test_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:33:42.082148Z","iopub.execute_input":"2022-07-24T21:33:42.082633Z","iopub.status.idle":"2022-07-24T21:33:42.555046Z","shell.execute_reply.started":"2022-07-24T21:33:42.082595Z","shell.execute_reply":"2022-07-24T21:33:42.554353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df = train_df_sample\ntest_df = test_df_sample","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:33:42.556941Z","iopub.execute_input":"2022-07-24T21:33:42.558445Z","iopub.status.idle":"2022-07-24T21:33:42.564211Z","shell.execute_reply.started":"2022-07-24T21:33:42.558403Z","shell.execute_reply":"2022-07-24T21:33:42.563164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del train_df_sample\ndel test_df_sample\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:33:48.640939Z","iopub.execute_input":"2022-07-24T21:33:48.642134Z","iopub.status.idle":"2022-07-24T21:33:49.127916Z","shell.execute_reply.started":"2022-07-24T21:33:48.642071Z","shell.execute_reply":"2022-07-24T21:33:49.126975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:34:22.121940Z","iopub.execute_input":"2022-07-24T21:34:22.122332Z","iopub.status.idle":"2022-07-24T21:34:22.148041Z","shell.execute_reply.started":"2022-07-24T21:34:22.122303Z","shell.execute_reply":"2022-07-24T21:34:22.146799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['customer_ID'] =\\\n    test_df['customer_ID'].apply(lambda x: int(x[-16:],16) ).astype('int64')\ntest_df['S_2'] = pd.to_datetime(test_df['S_2'])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:34:50.132126Z","iopub.execute_input":"2022-07-24T21:34:50.132492Z","iopub.status.idle":"2022-07-24T21:34:50.155356Z","shell.execute_reply.started":"2022-07-24T21:34:50.132464Z","shell.execute_reply":"2022-07-24T21:34:50.153755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# no intersection between the customer_IDs\nboolean_intersect = train['customer_ID'].isin(test_df['customer_ID'])\nnp.sum(boolean_intersect)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:36:43.440590Z","iopub.execute_input":"2022-07-24T21:36:43.440966Z","iopub.status.idle":"2022-07-24T21:36:43.448444Z","shell.execute_reply.started":"2022-07-24T21:36:43.440937Z","shell.execute_reply":"2022-07-24T21:36:43.447174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:34:55.283730Z","iopub.execute_input":"2022-07-24T21:34:55.284342Z","iopub.status.idle":"2022-07-24T21:34:55.304709Z","shell.execute_reply.started":"2022-07-24T21:34:55.284310Z","shell.execute_reply":"2022-07-24T21:34:55.304042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:17:04.698834Z","iopub.execute_input":"2022-07-24T20:17:04.699173Z","iopub.status.idle":"2022-07-24T20:17:04.720389Z","shell.execute_reply.started":"2022-07-24T20:17:04.699147Z","shell.execute_reply":"2022-07-24T20:17:04.719192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_labels = pd.read_csv('/kaggle/input/amex-default-prediction/train_labels.csv')  # gives TextFileReader\n# train_labels['customer_ID'] =\\\n#     train_labels['customer_ID'].apply(lambda x: int(x[-16:],16) ).astype('int64')","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:17:47.210131Z","iopub.execute_input":"2022-07-24T03:17:47.210480Z","iopub.status.idle":"2022-07-24T03:17:48.901289Z","shell.execute_reply.started":"2022-07-24T03:17:47.210444Z","shell.execute_reply":"2022-07-24T03:17:48.900120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_with_labels = pd.merge(train_df,train_labels,left_on='customer_ID',right_on='customer_ID',how='left')\n# del train_labels\n# del train_df\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:17:48.902476Z","iopub.execute_input":"2022-07-24T03:17:48.902759Z","iopub.status.idle":"2022-07-24T03:17:49.317666Z","shell.execute_reply.started":"2022-07-24T03:17:48.902730Z","shell.execute_reply":"2022-07-24T03:17:49.316724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df_with_labels[train_df_with_labels['customer_ID'] == 6642993746178481797][['customer_ID','S_2','D_43','S_3','S_7','target']].head(13)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:28:02.422828Z","iopub.execute_input":"2022-07-24T03:28:02.423210Z","iopub.status.idle":"2022-07-24T03:28:02.440238Z","shell.execute_reply.started":"2022-07-24T03:28:02.423183Z","shell.execute_reply":"2022-07-24T03:28:02.439017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pd.set_option('display.max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:31:49.975528Z","iopub.execute_input":"2022-07-24T03:31:49.975987Z","iopub.status.idle":"2022-07-24T03:31:49.981634Z","shell.execute_reply.started":"2022-07-24T03:31:49.975956Z","shell.execute_reply":"2022-07-24T03:31:49.980607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_with_labels[train_df_with_labels['customer_ID'] == 6642993746178481797].head(13)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:33:37.413234Z","iopub.execute_input":"2022-07-24T03:33:37.413620Z","iopub.status.idle":"2022-07-24T03:33:37.596494Z","shell.execute_reply.started":"2022-07-24T03:33:37.413594Z","shell.execute_reply":"2022-07-24T03:33:37.594845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import missingno as msno","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:23:30.354281Z","iopub.execute_input":"2022-07-24T03:23:30.354738Z","iopub.status.idle":"2022-07-24T03:23:31.301741Z","shell.execute_reply.started":"2022-07-24T03:23:30.354707Z","shell.execute_reply":"2022-07-24T03:23:31.300601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# msno.matrix(train_df_with_labels[col_for_view])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:26:16.896591Z","iopub.execute_input":"2022-07-24T03:26:16.897277Z","iopub.status.idle":"2022-07-24T03:26:17.479238Z","shell.execute_reply.started":"2022-07-24T03:26:16.897202Z","shell.execute_reply":"2022-07-24T03:26:17.477694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# spend_cols = []\n# for col in train_df_with_labels.columns[2:]:\n#     if col[:1] == \"S\":\n#         spend_cols.append(col)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:24:40.416832Z","iopub.execute_input":"2022-07-24T03:24:40.417233Z","iopub.status.idle":"2022-07-24T03:24:40.423416Z","shell.execute_reply.started":"2022-07-24T03:24:40.417203Z","shell.execute_reply":"2022-07-24T03:24:40.422316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# balance_cols = []\n# for col in train_df_with_labels.columns[2:]:\n#     if col[:1] == \"B\":\n#         balance_cols.append(col)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:02:46.432719Z","iopub.execute_input":"2022-07-24T04:02:46.433092Z","iopub.status.idle":"2022-07-24T04:02:46.439106Z","shell.execute_reply.started":"2022-07-24T04:02:46.433067Z","shell.execute_reply":"2022-07-24T04:02:46.437928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delinquency_cols = []\n# for col in train_df_with_labels.columns[2:]:\n#     if col[:1] == \"D\":\n#         delinquency_cols.append(col)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:03:19.175609Z","iopub.execute_input":"2022-07-24T04:03:19.175960Z","iopub.status.idle":"2022-07-24T04:03:19.182006Z","shell.execute_reply.started":"2022-07-24T04:03:19.175934Z","shell.execute_reply":"2022-07-24T04:03:19.180866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# defaults = train_df_with_labels[train_df_with_labels['target'] == 1]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:06:04.112986Z","iopub.execute_input":"2022-07-24T04:06:04.113314Z","iopub.status.idle":"2022-07-24T04:06:04.121819Z","shell.execute_reply.started":"2022-07-24T04:06:04.113289Z","shell.execute_reply":"2022-07-24T04:06:04.120641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# db_cols = delinquency_cols + balance_cols","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:07:05.620170Z","iopub.execute_input":"2022-07-24T04:07:05.620500Z","iopub.status.idle":"2022-07-24T04:07:05.625437Z","shell.execute_reply.started":"2022-07-24T04:07:05.620475Z","shell.execute_reply":"2022-07-24T04:07:05.624417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corr = defaults[db_cols].corr()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:08:59.170002Z","iopub.execute_input":"2022-07-24T04:08:59.170342Z","iopub.status.idle":"2022-07-24T04:08:59.256227Z","shell.execute_reply.started":"2022-07-24T04:08:59.170318Z","shell.execute_reply":"2022-07-24T04:08:59.254638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# corr[corr['D_42'] == -1]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:47:33.779351Z","iopub.execute_input":"2022-07-24T04:47:33.779740Z","iopub.status.idle":"2022-07-24T04:47:33.857773Z","shell.execute_reply.started":"2022-07-24T04:47:33.779714Z","shell.execute_reply":"2022-07-24T04:47:33.856685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_with_labels[['D_42','D_88']].head()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:47:59.560370Z","iopub.execute_input":"2022-07-24T04:47:59.560846Z","iopub.status.idle":"2022-07-24T04:47:59.574880Z","shell.execute_reply.started":"2022-07-24T04:47:59.560814Z","shell.execute_reply":"2022-07-24T04:47:59.573546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df_with_labels['D_88'].isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:48:43.924925Z","iopub.execute_input":"2022-07-24T04:48:43.925312Z","iopub.status.idle":"2022-07-24T04:48:43.934439Z","shell.execute_reply.started":"2022-07-24T04:48:43.925282Z","shell.execute_reply":"2022-07-24T04:48:43.933469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import seaborn as sns\n# %matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:08:23.479689Z","iopub.execute_input":"2022-07-24T04:08:23.480074Z","iopub.status.idle":"2022-07-24T04:08:23.488078Z","shell.execute_reply.started":"2022-07-24T04:08:23.480045Z","shell.execute_reply":"2022-07-24T04:08:23.487157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # plot the heatmap\n# sns.heatmap(corr.iloc[0:10,0:10])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T04:10:21.184012Z","iopub.execute_input":"2022-07-24T04:10:21.184427Z","iopub.status.idle":"2022-07-24T04:10:21.850267Z","shell.execute_reply.started":"2022-07-24T04:10:21.184396Z","shell.execute_reply":"2022-07-24T04:10:21.848778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create box-and-whisker plots and two-proportion z-test of differences between groups for D variables. Loop.\n\nFor defaults, check if there is a value or 4 values in the balance columns in the last month (or four months) for all customer_IDs.","metadata":{}},{"cell_type":"code","source":"# col_for_view = ['customer_ID','S_2'] + spend_cols","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:25:36.992510Z","iopub.execute_input":"2022-07-24T03:25:36.992883Z","iopub.status.idle":"2022-07-24T03:25:36.997333Z","shell.execute_reply.started":"2022-07-24T03:25:36.992857Z","shell.execute_reply":"2022-07-24T03:25:36.996391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# col_for_view","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:25:42.248131Z","iopub.execute_input":"2022-07-24T03:25:42.248602Z","iopub.status.idle":"2022-07-24T03:25:42.254375Z","shell.execute_reply.started":"2022-07-24T03:25:42.248546Z","shell.execute_reply":"2022-07-24T03:25:42.253758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(spend_cols)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T03:24:56.981155Z","iopub.execute_input":"2022-07-24T03:24:56.981498Z","iopub.status.idle":"2022-07-24T03:24:56.986174Z","shell.execute_reply.started":"2022-07-24T03:24:56.981472Z","shell.execute_reply":"2022-07-24T03:24:56.985588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GroupShuffleSplit","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:19:52.596315Z","iopub.execute_input":"2022-07-24T20:19:52.596684Z","iopub.status.idle":"2022-07-24T20:19:52.923397Z","shell.execute_reply.started":"2022-07-24T20:19:52.596656Z","shell.execute_reply":"2022-07-24T20:19:52.922374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# doesn't actually have the labels yet due to RAM constraints\ntrain_df_with_labels = train_df\ndel train_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:20:25.835840Z","iopub.execute_input":"2022-07-24T20:20:25.836185Z","iopub.status.idle":"2022-07-24T20:20:25.964528Z","shell.execute_reply.started":"2022-07-24T20:20:25.836161Z","shell.execute_reply":"2022-07-24T20:20:25.963524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://stackoverflow.com/questions/54797508/how-to-generate-a-train-test-split-based-on-a-group-id\nsplitter = GroupShuffleSplit(test_size=.20, n_splits=1, random_state = 7)\nsplit = splitter.split(train_df_with_labels, groups=train_df_with_labels['customer_ID'])\ntrain_inds, test_inds = next(split)\n\ntrain = train_df_with_labels.iloc[train_inds]\n\n# this is right (we're using the provided data (our \"training data\") to make a training set and testing set)\ntest = train_df_with_labels.iloc[test_inds]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:20:31.327388Z","iopub.execute_input":"2022-07-24T20:20:31.327717Z","iopub.status.idle":"2022-07-24T20:20:31.346175Z","shell.execute_reply.started":"2022-07-24T20:20:31.327693Z","shell.execute_reply":"2022-07-24T20:20:31.345068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Check if you lost customer_ID here**","metadata":{}},{"cell_type":"code","source":"del train_inds\ndel test_inds\ndel train_df_with_labels\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:20:38.784354Z","iopub.execute_input":"2022-07-24T20:20:38.784700Z","iopub.status.idle":"2022-07-24T20:20:38.978460Z","shell.execute_reply.started":"2022-07-24T20:20:38.784672Z","shell.execute_reply":"2022-07-24T20:20:38.977739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_train = train.target\n# y_valid = test.target\n# train.drop(['target'], axis=1, inplace=True)\n# test.drop(['target'], axis=1, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/parulpandey/a-guide-to-handling-missing-values-in-python\ndef missing_values_table(df):\n        # Total missing values\n        mis_val = df.isnull().sum()\n        \n        # Percentage of missing values\n        mis_val_percent = 100 * df.isnull().sum() / len(df)\n        \n        # Make a table with the results\n        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n        \n        # Rename the columns\n        mis_val_table_ren_columns = mis_val_table.rename(\n        columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n        \n        # Sort the table by percentage of missing descending\n        mis_val_table_ren_columns = mis_val_table_ren_columns[\n            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n        '% of Total Values', ascending=False).round(1)\n        \n        # Print some summary information\n        print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n            \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n              \" columns that have missing values.\")\n        \n        # Return the dataframe with missing information\n        return mis_val_table_ren_columns","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:21:03.312653Z","iopub.execute_input":"2022-07-24T20:21:03.312949Z","iopub.status.idle":"2022-07-24T20:21:03.320967Z","shell.execute_reply.started":"2022-07-24T20:21:03.312926Z","shell.execute_reply":"2022-07-24T20:21:03.319465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# keep the columns from train that had no missing values\ncolumns_to_keep = train.columns[~train.isnull().any()]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:21:12.389502Z","iopub.execute_input":"2022-07-24T20:21:12.389823Z","iopub.status.idle":"2022-07-24T20:21:12.396956Z","shell.execute_reply.started":"2022-07-24T20:21:12.389799Z","shell.execute_reply":"2022-07-24T20:21:12.395881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(columns_to_keep))","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:21:14.530133Z","iopub.execute_input":"2022-07-24T20:21:14.530443Z","iopub.status.idle":"2022-07-24T20:21:14.535577Z","shell.execute_reply.started":"2022-07-24T20:21:14.530420Z","shell.execute_reply":"2022-07-24T20:21:14.534695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values_df = missing_values_table(train)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:32:04.223531Z","iopub.execute_input":"2022-07-24T20:32:04.224964Z","iopub.status.idle":"2022-07-24T20:32:04.250710Z","shell.execute_reply.started":"2022-07-24T20:32:04.224920Z","shell.execute_reply":"2022-07-24T20:32:04.249549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# columns that have at least 40% non-null values\n# cols_with_enough_data = missing_values_df[missing_values_df['% of Total Values'] < 60.0].index\ncols_with_enough_data = missing_values_df.index\n\n#print(columns_to_keep)\n#print(cols_with_enough_data)\ncolumns_to_keep_v2 = list(columns_to_keep)\ncolumns_to_keep_v2.extend(list(cols_with_enough_data))\n\nfinal_cols = list(set(columns_to_keep_v2))\nprint(len(final_cols))\nprint(final_cols)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:33:48.108034Z","iopub.execute_input":"2022-07-24T20:33:48.108437Z","iopub.status.idle":"2022-07-24T20:33:48.115775Z","shell.execute_reply.started":"2022-07-24T20:33:48.108407Z","shell.execute_reply":"2022-07-24T20:33:48.114278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = train[final_cols]\n# test = test[final_cols]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the problem is I'm iterating forward while removing elements in the array\n# explicit_cat = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\n# for col in explicit_cat:\n#     if col not in cols_with_enough_data:\n#         explicit_cat.remove(col)\n#         print(col)\n    \n# can probably shorten this to : explicit_cat = train.select_dtypes('category').columns\nexplicit_cat = train.select_dtypes('category').columns","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:35:32.418300Z","iopub.execute_input":"2022-07-24T20:35:32.418620Z","iopub.status.idle":"2022-07-24T20:35:32.425703Z","shell.execute_reply.started":"2022-07-24T20:35:32.418596Z","shell.execute_reply":"2022-07-24T20:35:32.425058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**To-Do**: need to handle categorical variables better","metadata":{}},{"cell_type":"code","source":"# Source: https://www.statology.org/convert-categorical-variable-to-numeric-pandas/\n#convert all categorical variables to numeric\n# should probably dummy this after factorizing\n# train[explicit_cat] = train[explicit_cat].apply(lambda x: pd.factorize(x)[0])\n# train[explicit_cat] = train[explicit_cat].astype('int8')\n\n# figure out how pd.factorize() treats nulls by testing your hypothesis on the dataset\n# test[explicit_cat] = test[explicit_cat].apply(lambda x: pd.factorize(x)[0])\n# test[explicit_cat] = test[explicit_cat].astype('int8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_missing_vals = missing_values_table(train)\ntrain_missing_vals","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:40:32.999353Z","iopub.execute_input":"2022-07-24T20:40:32.999787Z","iopub.status.idle":"2022-07-24T20:40:33.029673Z","shell.execute_reply.started":"2022-07-24T20:40:32.999756Z","shell.execute_reply":"2022-07-24T20:40:33.028817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.fillna.html\ncolumns_with_missing_vals = train_missing_vals.index\ntrain[columns_with_missing_vals] = train[columns_with_missing_vals].astype(float)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:40:49.367740Z","iopub.execute_input":"2022-07-24T20:40:49.368095Z","iopub.status.idle":"2022-07-24T20:40:49.401847Z","shell.execute_reply.started":"2022-07-24T20:40:49.368070Z","shell.execute_reply":"2022-07-24T20:40:49.400821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train[\"D_43\"] = train.groupby(\"customer_ID\")[\"D_43\"].transform(lambda x: x.fillna(method='ffill'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find the columns in the dataframe that have values less than -2 and more than 3 and\n# do something about them\n# maybe standardized them, maybe remove them, maybe encode them\n# sorted(train.describe().iloc[-1])","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:51:53.412356Z","iopub.execute_input":"2022-07-24T20:51:53.412696Z","iopub.status.idle":"2022-07-24T20:51:53.418520Z","shell.execute_reply.started":"2022-07-24T20:51:53.412671Z","shell.execute_reply":"2022-07-24T20:51:53.417163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:45:59.330452Z","iopub.execute_input":"2022-07-24T20:45:59.330769Z","iopub.status.idle":"2022-07-24T20:45:59.762037Z","shell.execute_reply.started":"2022-07-24T20:45:59.330745Z","shell.execute_reply":"2022-07-24T20:45:59.760789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train[columns_with_missing_vals] = train.groupby(\"customer_ID\")[columns_with_missing_vals].transform(lambda x: x.fillna(method='ffill'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train[['customer_ID','S_2','D_43','S_7']].iloc[100:120,:]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:07:05.031394Z","iopub.execute_input":"2022-07-24T21:07:05.031710Z","iopub.status.idle":"2022-07-24T21:07:05.036236Z","shell.execute_reply.started":"2022-07-24T21:07:05.031687Z","shell.execute_reply":"2022-07-24T21:07:05.035387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creates new columns with imputed null values\nfor col in columns_with_missing_vals:\n    train[col + '_fillna-2'] = train[col].fillna(-2)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T20:56:24.740564Z","iopub.execute_input":"2022-07-24T20:56:24.740889Z","iopub.status.idle":"2022-07-24T20:56:24.775407Z","shell.execute_reply.started":"2022-07-24T20:56:24.740866Z","shell.execute_reply":"2022-07-24T20:56:24.774048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# creates new columns for columns_with_missing_vals represented as booleans\nfor col in columns_with_missing_vals:\n    train[col + '_was_missing'] = train[col].isnull()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:03:46.199384Z","iopub.execute_input":"2022-07-24T21:03:46.199777Z","iopub.status.idle":"2022-07-24T21:03:46.236182Z","shell.execute_reply.started":"2022-07-24T21:03:46.199749Z","shell.execute_reply":"2022-07-24T21:03:46.235418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a new column using forward fill\nfor col in columns_with_missing_vals:\n    train[col + '_filna_ffill'] = train.groupby(\"customer_ID\")[col].transform(lambda x: x.fillna(method='ffill'))","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:15:02.298601Z","iopub.execute_input":"2022-07-24T21:15:02.298971Z","iopub.status.idle":"2022-07-24T21:15:10.767320Z","shell.execute_reply.started":"2022-07-24T21:15:02.298945Z","shell.execute_reply":"2022-07-24T21:15:10.766277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# using fillna_ffill columns, creates new columns with imputed null values\nfor col in columns_with_missing_vals:\n    train[col + '_filna_ffill_fillna-2'] = train[col + '_filna_ffill'].fillna(-2)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:21:46.113529Z","iopub.execute_input":"2022-07-24T21:21:46.113934Z","iopub.status.idle":"2022-07-24T21:21:46.148225Z","shell.execute_reply.started":"2022-07-24T21:21:46.113905Z","shell.execute_reply":"2022-07-24T21:21:46.147604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# using fillna_ffill columns, creates new columns represented as booleans\nfor col in columns_with_missing_vals:\n    train[col + '_filna_ffill_was_missing'] = train[col + '_filna_ffill'].isnull()","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:23:57.725680Z","iopub.execute_input":"2022-07-24T21:23:57.725986Z","iopub.status.idle":"2022-07-24T21:23:57.761389Z","shell.execute_reply.started":"2022-07-24T21:23:57.725963Z","shell.execute_reply":"2022-07-24T21:23:57.760281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# need to drop columns with missing vals\nfor col in columns_with_missing_vals:\n    train.drop(axis=1, columns=[col, col + '_filna_ffill'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:30:04.786140Z","iopub.execute_input":"2022-07-24T21:30:04.786552Z","iopub.status.idle":"2022-07-24T21:30:05.032005Z","shell.execute_reply.started":"2022-07-24T21:30:04.786521Z","shell.execute_reply":"2022-07-24T21:30:05.030807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# no column in train should have null values at this point\n# check to see if this is true\nnp.sum(train.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:31:15.945692Z","iopub.execute_input":"2022-07-24T21:31:15.946077Z","iopub.status.idle":"2022-07-24T21:31:15.959781Z","shell.execute_reply.started":"2022-07-24T21:31:15.946048Z","shell.execute_reply":"2022-07-24T21:31:15.958946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train[['customer_ID','D_43']].iloc[10:30,:]","metadata":{"execution":{"iopub.status.busy":"2022-07-24T21:25:47.931488Z","iopub.execute_input":"2022-07-24T21:25:47.931811Z","iopub.status.idle":"2022-07-24T21:25:47.936040Z","shell.execute_reply.started":"2022-07-24T21:25:47.931787Z","shell.execute_reply":"2022-07-24T21:25:47.934973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_missing_vals = missing_values_table(test)\ntest_missing_vals","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.fillna.html\ncolumns_with_missing_vals = test_missing_vals.index\n\n# the line below should work as we used pd.factorize() above to convert categorical columns to numerical ones\n# TO-DO: make sure you're grouping and then filling, as if you don't filling may occur across\n# customer_IDs\n# Source: https://stackoverflow.com/questions/19966018/pandas-filling-missing-values-by-mean-in-each-group\n# This can be accomplished likely by the first line of code below:\n# df[\"value\"] = df.groupby(\"name\")[\"value\"].transform(lambda x: x.fillna(x.mean()))\n# df['value'] = df['value'].fillna(df.groupby('name')['value'].transform('mean'))\ntest[columns_with_missing_vals] = test[columns_with_missing_vals].astype(float)\ntest[columns_with_missing_vals] = test[columns_with_missing_vals].fillna(method='ffill')\ntest[columns_with_missing_vals] = test[columns_with_missing_vals].fillna(-2)\ntest.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You lost the customer ID and dates at some point above, which you may need to recover.","metadata":{}},{"cell_type":"code","source":"# converts float64 to float32\ntrain[train.select_dtypes('float64').columns] = train.select_dtypes('float64').astype('float32')\ntest[test.select_dtypes('float64').columns] = test.select_dtypes('float64').astype('float32')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"python pipelines and column transformers making your own custom (preprocessing) transformers\n\ni. https://towardsdatascience.com/custom-transformers-and-ml-data-pipelines-with-python-20ea2a7adb65 - **Look at these steps again**!\n\nii. https://machinelearningmastery.com/how-to-transform-target-variables-for-regression-with-scikit-learn","metadata":{}},{"cell_type":"code","source":"# Source: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.fillna.html\n# Make sure the way you process the test data is the same as the training data\n# columns_with_missing_vals = train_missing_vals.index\n# test[columns_with_missing_vals] = test[columns_with_missing_vals].astype(float)\n# test[columns_with_missing_vals] = test[columns_with_missing_vals].fillna(method='ffill')\n# test[columns_with_missing_vals] = test[columns_with_missing_vals].fillna(-2)\n# test.head(20)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import confusion_matrix, classification_report, mean_absolute_error","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"important_cols = ['P_2','D_48','D_77','R_27','D_52','S_3']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Index(['P_2', 'D_48', 'R_27', 'D_77', 'D_52', 'S_3', 'D_50', 'D_43', 'D_105',\n       'D_121', 'B_25', 'D_56', 'D_55', 'D_46', 'S_7', 'D_69', 'D_115', 'D_61',\n       'D_119', 'B_13', 'D_118', 'D_62', 'S_9', 'B_15', 'D_104', 'B_17',\n       'S_27', 'S_22', 'P_3', 'S_26', 'S_24', 'D_102', 'D_141', 'D_133', 'B_8',\n       'S_23', 'S_25', 'D_144', 'D_128', 'D_112', 'D_131', 'D_130'],\n      dtype='object')","metadata":{}},{"cell_type":"markdown","source":"**HALT:** BEFORE RUNNING THE MODEL, MAKE SURE YOU HAVE REMOVED HIGHLY CORRELATED FEATURES.","metadata":{}},{"cell_type":"code","source":"forest_model = RandomForestRegressor(random_state=1)\nforest_model.fit(train, y_train)\npredictions = forest_model.predict(test)\nprint(mean_absolute_error(y_valid, predictions))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feats = {} # a dict to hold feature_name: feature_importance\nfor feature, importance in zip(train.columns, forest_model.feature_importances_):\n    feats[feature] = importance #add the name/value pair \n\nimportances = pd.DataFrame.from_dict(feats, orient='index').rename(columns={0: 'Gini-importance'})\nimportances.sort_values(by='Gini-importance').plot(kind='bar', rot=45)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"importances.sort_values(by='Gini-importance', ascending=False).head(15)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"0.2530049261083744","metadata":{}},{"cell_type":"code","source":"predictions_description = pd.DataFrame(predictions).describe()\npredictions_description","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"median = predictions_description.reset_index().iloc[5][0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://github.ccs.neu.edu/alexandrews248/FINA4390/blob/main/Mini_Project_5.ipynb\n# can probably use a form of grid search to determine the most optimal value\ny_decisions = [0 if x < median else 1 for x in predictions]\n\ndf = pd.DataFrame({'Actual y':y_valid,'Predicted y':y_decisions,'Predicted prob y is 1':predictions})\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_df = pd.DataFrame(confusion_matrix(df['Actual y'], df['Predicted y']), columns=['Predicted group ' + str(cat) for cat in [0,1]], index=['Actual group ' + str(cat) for cat in [0,1]])\nconfusion_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_metric_numpy(np.array(df['Actual y']), \n                  np.array(df['Predicted y']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions = forest_model.predict(test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The results above are fairly good. But remember: you partitioned the train set into two parts. The first part you trained your model, and the second part was used to validate the training. Now, you must train your model on all of the data (in other words, the original train set, pre-partition).\n\nYou also need to figure out a way to get the customer IDs - just get the last 16 bytes as before and create a table mapping the strings to integers\n\nTraining and inference pipeline using a large dataset: https://www.kaggle.com/competitions/amex-default-prediction/discussion/327228 (you just need to submit working code - you may not need to see how the model works on the entirety of the dataset, just submit it)","metadata":{"execution":{"iopub.status.busy":"2022-07-18T15:54:54.915603Z","iopub.execute_input":"2022-07-18T15:54:54.916131Z","iopub.status.idle":"2022-07-18T15:54:54.932536Z","shell.execute_reply.started":"2022-07-18T15:54:54.916092Z","shell.execute_reply":"2022-07-18T15:54:54.931225Z"}}},{"cell_type":"markdown","source":"In other words, your submission will only include one instance of each customer_ID (not multiple instances of the same customer_ID). The code below should handle this issue","metadata":{}},{"cell_type":"markdown","source":"Also, figure out how to get back the original customer_IDs for submission file - justs create table, as mentioned above\n\nPrivate lb is from the ones with the last statement in 2019 October.\n\nIf the customer does not pay due amount in 120 days after their latest statement date it is considered a default event. - Do Feature Engineering on the B features (balance) and use the last statement values (groupby(customer_ID).tail(1)). Create lag features and shift 3-5 months.","metadata":{}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/inversion/amex-competition-metric-python\n\n# ave_p2 = (train_data\n#           .groupby('customer_ID')\n#           .mean()\n#           .rename(columns={'P_2': 'prediction'}))\n\n# # Scale the mean P_2 by the max value and take the compliment\n# ave_p2['prediction'] = 1.0 - (ave_p2['prediction'] / ave_p2['prediction'].max())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/amex-default-prediction/discussion/327094\n# Here's a way using pandas to just keep the last statement month per customer.\n# X_train =  (train_data\n#             .groupby('customer_ID')\n#             .tail(1)\n#             .set_index('customer_ID', drop=True)\n#             .sort_index()\n#             .fillna(-999)\n#             .drop(['S_2'], axis='columns'))\n\n# similar to:\n# train.groupby('customer_ID').tail(1).drop(['S_2'], axis='columns')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Alternative: keeping the last month per customer\n# X_train = X_train.drop_duplicates(subset=[\"customer_ID\"], keep=\"last\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/amex-default-prediction/discussion/327361\n# train.drop_duplicates(subset=['customer_ID'], keep='last').drop(['S_2'], axis='columns')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/ieee-fraud-detection/discussion/111245\n# creates all feature interactions\nfor i in range(len(cols)):\n    for j in range(i + 1, len(cols)):\n        X['{}_div_{}'.format(cols[i], cols[j])] = X[cols[i]] / X[cols[j]]\n        X['{}_mul_{}'.format(cols[i], cols[j])] = X[cols[i]] * X[cols[j]]\n        X['{}_sub1_{}'.format(cols[i], cols[j])] = X[cols[i]] - X[cols[j]]\n        X['{}_sub2_{}'.format(cols[j], cols[i])] = X[cols[j]] - X[cols[i]]\n        X['{}_add_{}'.format(cols[i], cols[j])] = X[cols[i]] + X[cols[j]]\n        test['{}_div_{}'.format(c_cols[i], c_cols[j])] = test[c_cols[i]] / test[c_cols[j]]\n        test['{}_mul_{}'.format(c_cols[i], c_cols[j])] = test[c_cols[i]] * test[c_cols[j]]\n        test['{}_sub1_{}'.format(c_cols[i], c_cols[j])] = test[c_cols[i]] - test[c_cols[j]]\n        test['{}_sub2_{}'.format(c_cols[j], c_cols[i])] = test[c_cols[j]] - test[c_cols[i]]\n\n# compute a correlation matrix for all the generated features and\n# remove features with a high correlation\n# bruteforce_cols = [col for col in X.columns if '_mul_' in col or '_sub_' in col or '_div_' in col or '_add_' in col]\n# corr_matrix = X[bruteforce_cols].corr()\n\n# to_drop = list()\n\n# for i in range(1, len(corr_matrix)):\n#     for j in range(i):\n#         # See if the correlation between two features are more than a selected threshold\n#         if corr_matrix.iloc[i, j] >= 0.98:\n#             # Then keep the one from thos two which correlates with target better\n#             if abs(pd.concat([X[corr_matrix.index[i]], y], axis=1).corr().iloc[0][1]) > abs(pd.concat([X[corr_matrix.columns[j]], y], axis=1).corr().iloc[0][1]):\n#                 to_drop.append(corr_matrix.columns[j])\n#             else:\n#                 to_drop.append(corr_matrix.index[i])\n\n# to_drop = list(set(to_drop))\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"XGBoost [Starter](https://www.kaggle.com/code/cdeotte/xgboost-starter-0-793) (79.3%).","metadata":{}},{"cell_type":"code","source":"# When inferring test data in a Kaggle notebook, \n# we can read a few million rows at a time and therefore read, \n# process, and infer the test data in parts.\nall_preds = []\nfor parts in range(NUM_PARTS):\n    test_part = pd.read_csv('test', nrows = ROWS, skiprows = SKIP)\n    test_part = process(test_part)\n    all_preds.append( infer(test_part) )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\"Splitting notebooks into training and inference is also the easiest way to expand on those already excellent notebooks.\"\n\nAn Overview of Encoding [Technqiues](https://www.kaggle.com/code/shahules/an-overview-of-encoding-techniques/notebook).\n\nIntroduction to Manual Feature [Engineering](https://www.kaggle.com/willkoehrsen/introduction-to-manual-feature-engineering).\n\nAutomated Feature Engineering [Basics](https://www.kaggle.com/code/willkoehrsen/automated-feature-engineering-basics/notebook).\n\n6 Ways for Feature [Selection](https://www.kaggle.com/code/sz8416/6-ways-for-feature-selection/comments).\n\nGood [Approach](https://www.kaggle.com/competitions/amex-default-prediction/discussion/336546).\n\nIEEE Fradulent Client [Prediction](https://www.kaggle.com/code/alijs1/ieee-transaction-columns-reference/notebook). - you should view this competition as predicting clients that will default, not transactions that will default\n\nTry many different validation strategies (train #, skip #, predict #)\n\nEnsure features in the training set come from the same distribution as the test set using code like. *scipy.stats import ks_2sam*.\n\nChris Winning Notebook in IEEE Fraud [Competition](https://www.kaggle.com/code/cdeotte/xgb-fraud-with-magic-0-9600/notebook).\n\nConsider using custom objective and evaluation metric directly in [models](https://xgboost.readthedocs.io/en/stable/tutorials/custom_metric_obj.html).","metadata":{}},{"cell_type":"code","source":"# Frequency Encoding\n# temp = df['card1'].value_counts().to_dict()\n# df['card1_counts'] = df['card1'].map(temp)\n\n# Aggregations / Group Statistics\n# temp = df.groupby('card1')['TransactionAmt'].agg(['mean'])   \n#     .rename({'mean':'TransactionAmt_card1_mean'},axis=1)\n# df = pd.merge(df,temp,on='card1',how='left')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_features = df.groupby('uid')[CM_columns].agg(['mean'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predicting on Original TEST Set","metadata":{}},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"customer_date  = test_df[['customer_ID', 'S_2']].copy()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Write Submission File","metadata":{}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/cdeotte/xgboost-starter-0-793\n# WRITE SUBMISSION FILE\ntest_preds = np.concatenate(test_preds)\ntest = cudf.DataFrame(index=customers,data={'prediction':test_preds})\nsub = cudf.read_csv('../input/amex-default-prediction/sample_submission.csv')[['customer_ID']]\nsub['customer_ID_hash'] = sub['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\nsub = sub.set_index('customer_ID_hash')\nsub = sub.merge(test[['prediction']], left_index=True, right_index=True, how='left')\nsub = sub.reset_index(drop=True)\n\n# DISPLAY SUBMISSION FILE\nsub.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# End","metadata":{}},{"cell_type":"markdown","source":"# Custom Transformer Example (BEGIN)","metadata":{}},{"cell_type":"code","source":"# Feature Selector\nimport numpy as np \nimport pandas as pd\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import FeatureUnion, Pipeline \n\n#Custom Transformer that extracts columns passed as argument to its constructor \nclass FeatureSelector( BaseEstimator, TransformerMixin ):\n    #Class Constructor \n    def __init__( self, feature_names ):\n        self._feature_names = feature_names \n    \n    #Return self nothing else to do here    \n    def fit( self, X, y = None ):\n        return self \n    \n    #Method that describes what we need this transformer to do\n    def transform( self, X, y = None ):\n        return X[ self._feature_names ] ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Categorical Transformer\n#Custom transformer that breaks dates column into year, month and day into separate columns and\n#converts certain features to binary \nclass CategoricalTransformer( BaseEstimator, TransformerMixin ):\n    #Class constructor method that takes in a list of values as its argument\n    def __init__(self, use_dates = ['year', 'month', 'day'] ):\n        self._use_dates = use_dates\n        \n    #Return self nothing else to do here\n    def fit( self, X, y = None  ):\n        return self\n\n    #Helper function to extract year from column 'dates' \n    def get_year( self, obj ):\n        return str(obj)[:4]\n    \n    #Helper function to extract month from column 'dates'\n    def get_month( self, obj ):\n        return str(obj)[4:6]\n    \n    #Helper function to extract day from column 'dates'\n    def get_day(self, obj):\n        return str(obj)[6:8]\n    \n    #Helper function that converts values to Binary depending on input \n    def create_binary(self, obj):\n        if obj == 0:\n            return 'No'\n        else:\n            return 'Yes'\n    \n    #Transformer method we wrote for this transformer \n    def transform(self, X , y = None ):\n       #Depending on constructor argument break dates column into specified units\n       #using the helper functions written above \n        for spec in self._use_dates:\n            exec( \"X.loc[:,'{}'] = X['date'].apply(self.get_{})\".format( spec, spec ) )\n       #Drop unusable column \n        X = X.drop('date', axis = 1 )\n       \n       #Convert these columns to binary for one-hot-encoding later\n        X.loc[:,'waterfront'] = X['waterfront'].apply( self.create_binary )\n       \n        X.loc[:,'view'] = X['view'].apply( self.create_binary )\n       \n        X.loc[:,'yr_renovated'] = X['yr_renovated'].apply( self.create_binary )\n       #returns numpy array\n        return X.values ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Numercial Transformer\n#Custom transformer we wrote to engineer features ( bathrooms per bedroom and/or how old the house is in 2019  ) \n#passed as boolen arguements to its constructor\nclass NumericalTransformer(BaseEstimator, TransformerMixin):\n    #Class Constructor\n    def __init__( self, bath_per_bed = True, years_old = True ):\n        self._bath_per_bed = bath_per_bed\n        self._years_old = years_old\n        \n    #Return self, nothing else to do here\n    def fit( self, X, y = None ):\n        return self \n    \n    #Custom transform method we wrote that creates aformentioned features and drops redundant ones \n    def transform(self, X, y = None):\n        #Check if needed \n        if self._bath_per_bed:\n            #create new column\n            X.loc[:,'bath_per_bed'] = X['bathrooms'] / X['bedrooms']\n            #drop redundant column\n            X.drop('bathrooms', axis = 1 )\n        #Check if needed     \n        if self._years_old:\n            #create new column\n            X.loc[:,'years_old'] =  2019 - X['yr_built']\n            #drop redundant column \n            X.drop('yr_built', axis = 1)\n            \n        #Converting any infinity values in the dataset to Nan\n        X = X.replace( [ np.inf, -np.inf ], np.nan )\n        #returns a numpy array\n        return X.values","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Combining Pipelines Together\n#Categrical features to pass down the categorical pipeline \ncateforical_features = ['date', 'waterfront', 'view', 'yr_renovated']\n\n#Numerical features to pass down the numerical pipeline \nnumerical_features = ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors',\n                'condition', 'grade', 'sqft_basement', 'yr_built']\n\n#Defining the steps in the categorical pipeline \ncategorical_pipeline = Pipeline( steps = [ ( 'cat_selector', FeatureSelector(categorical_features) ),\n                                  \n                                  ( 'cat_transformer', CategoricalTransformer() ), \n                                  \n                                  ( 'one_hot_encoder', OneHotEncoder( sparse = False ) ) ] )\n    \n#Defining the steps in the numerical pipeline     \nnumerical_pipeline = Pipeline( steps = [ ( 'num_selector', FeatureSelector(numerical_features) ),\n                                  \n                                  ( 'num_transformer', NumericalTransformer() ),\n                                  \n                                  ('imputer', SimpleImputer(strategy = 'median') ),\n                                  \n                                  ( 'std_scaler', StandardScaler() ) ] )\n\n#Combining numerical and categorical piepline into one full big pipeline horizontally \n#using FeatureUnion\nfull_pipeline = FeatureUnion( transformer_list = [ ( 'categorical_pipeline', categorical_pipeline ), \n                                                  \n                                                  ( 'numerical_pipeline', numerical_pipeline ) ] )","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# including the ML model\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import train_test_split\n\n#Leave it as a dataframe becuase our pipeline is called on a \n#pandas dataframe to extract the appropriate columns, remember?\nX = data.drop('price', axis = 1)\n#You can covert the target variable to numpy \ny = data['price'].values \n\nX_train, X_test, y_train, y_test = train_test_split( X, y , test_size = 0.2 , random_state = 42 )\n\n#The full pipeline as a step in another pipeline with an estimator as the final step\nfull_pipeline_m = Pipeline( steps = [ ( 'full_pipeline', full_pipeline),\n                                  \n                                  ( 'model', LinearRegression() ) ] )\n\n#Can call fit on it just like any other pipeline\nfull_pipeline_m.fit( X_train, y_train )\n\n#Can predict with it like any other pipeline\ny_pred = full_pipeline_m.predict( X_test ) ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Custom Transformer Example (END)","metadata":{}},{"cell_type":"code","source":"test_df.head(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**To-Do**: Continue Preprocessing | Check out Pandas CSV chunker (create many CSV files and store them on Kaggle to reduce RAM usage) | complete the last part of [step 1](https://www.kaggle.com/competitions/amex-default-prediction/discussion/328054).\n\nUsing my Housing Prices Competition notebook as inspiration for preprocessing - https://www.kaggle.com/code/greenfruit2/exercise-xgboost?scriptVersionId=82622304","metadata":{}},{"cell_type":"code","source":"train[train['D_82'].isna()].head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_full[['customer_ID','S_2','D_76']].head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select categorical columns with relatively low cardinality (convenient but arbitrary)\n# Get low_cardinality_cols that are non_numeric\nlow_cardinality_cols = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get numeric columns\nnumeric_columns = []","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_sample.info(memory_usage='deep')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in cat_cols:\n    tmp_num = len(train_df[col].unique())\n    print(f\"Number of unique values for column: {col} is: \\t {tmp_num}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[f'Number of unique values for column: {col} is:\\t {len(train_df[col].unique())}' for col in cat_cols]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[f'Unique values for column: {col} are: {list(train_df[col].unique())}' for col in cat_cols]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* https://www.kaggle.com/code/rohanrao/tutorial-on-reading-large-datasets/notebook\n* Parquet Format Dataset for Low Memory [Use](https://www.kaggle.com/competitions/amex-default-prediction/discussion/327138). ","metadata":{}},{"cell_type":"code","source":"# #  defining data types\n# # Source: https://www.kaggle.com/code/abhianalytic/amex-read-full-data-without-any-transformation\n\n# # Source: https://www.kaggle.com/competitions/amex-default-prediction/discussion/328054\n# # Column S_2: This column is a date with time.\n# # This column is provided as a string of length 10 which uses 10 bytes per row! This is too much! \n# #If we convert this column with pd.to_datetime() then it becomes only 4 bytes. \n# #Or we can save this column as three columns of year_last_2_digits, month, and day as int8 each and \n# # only use 3 bytes per row.\n# dtypes = {'customer_ID':'O',\n# 'S_2':'O',\n# 'P_2':'float32',\n# 'D_39':'float32',\n# 'B_1':'float32',\n# 'B_2':'float32',\n# 'R_1':'float32',\n# 'S_3':'float32',\n# 'D_41':'float32',\n# 'B_3':'float32',\n# 'D_42':'float32',\n# 'D_43':'float32',\n# 'D_44':'float32',\n# 'B_4':'float32',\n# 'D_45':'float32',\n# 'B_5':'float32',\n# 'R_2':'float32',\n# 'D_46':'float32',\n# 'D_47':'float32',\n# 'D_48':'float32',\n# 'D_49':'float32',\n# 'B_6':'float32',\n# 'B_7':'float32',\n# 'B_8':'float32',\n# 'D_50':'float32',\n# 'D_51':'float32',\n# 'B_9':'float32',\n# 'R_3':'float32',\n# 'D_52':'float32',\n# 'P_3':'float32',\n# 'B_10':'float32',\n# 'D_53':'float32',\n# 'S_5':'float32',\n# 'B_11':'float32',\n# 'S_6':'float32',\n# 'D_54':'float32',\n# 'R_4':'float32',\n# 'S_7':'float32',\n# 'B_12':'float32',\n# 'S_8':'float32',\n# 'D_55':'float32',\n# 'D_56':'float32',\n# 'B_13':'float32',\n# 'R_5':'float32',\n# 'D_58':'float32',\n# 'S_9':'float32',\n# 'B_14':'float32',\n# 'D_59':'float32',\n# 'D_60':'float32',\n# 'D_61':'float32',\n# 'B_15':'float32',\n# 'S_11':'float32',\n# 'D_62':'float32',\n# 'D_63':'category',\n# 'D_64':'category',\n# 'D_65':'float32',\n# 'B_16':'float32',\n# 'B_17':'float32',\n# 'B_18':'float32',\n# 'B_19':'float32',\n# 'D_66':'category',\n# 'B_20':'float32',\n# 'D_68':'category',\n# 'S_12':'float32',\n# 'R_6':'float32',\n# 'S_13':'float32',\n# 'B_21':'float32',\n# 'D_69':'float32',\n# 'B_22':'float32',\n# 'D_70':'float32',\n# 'D_71':'float32',\n# 'D_72':'float32',\n# 'S_15':'float32',\n# 'B_23':'float32',\n# 'D_73':'float32',\n# 'P_4':'float32',\n# 'D_74':'float32',\n# 'D_75':'float32',\n# 'D_76':'float32',\n# 'B_24':'float32',\n# 'R_7':'float32',\n# 'D_77':'float32',\n# 'B_25':'float32',\n# 'B_26':'float32',\n# 'D_78':'float32',\n# 'D_79':'float32',\n# 'R_8':'float32',\n# 'R_9':'float32',\n# 'S_16':'float32',\n# 'D_80':'float32',\n# 'R_10':'float32',\n# 'R_11':'float32',\n# 'B_27':'float32',\n# 'D_81':'float32',\n# 'D_82':'float32',\n# 'S_17':'float32',\n# 'R_12':'float32',\n# 'B_28':'float32',\n# 'R_13':'float32',\n# 'D_83':'float32',\n# 'R_14':'float32',\n# 'R_15':'float32',\n# 'D_84':'float32',\n# 'R_16':'float32',\n# 'B_29':'float32',\n# 'B_30':'category',\n# 'S_18':'float32',\n# 'D_86':'float32',\n# 'D_87':'float32',\n# 'R_17':'float32',\n# 'R_18':'float32',\n# 'D_88':'float32',\n# 'B_31':'int32',\n# 'S_19':'float32',\n# 'R_19':'float32',\n# 'B_32':'float32',\n# 'S_20':'float32',\n# 'R_20':'float32',\n# 'R_21':'float32',\n# 'B_33':'float32',\n# 'D_89':'float32',\n# 'R_22':'float32',\n# 'R_23':'float32',\n# 'D_91':'float32',\n# 'D_92':'float32',\n# 'D_93':'float32',\n# 'D_94':'float32',\n# 'R_24':'float32',\n# 'R_25':'float32',\n# 'D_96':'float32',\n# 'S_22':'float32',\n# 'S_23':'float32',\n# 'S_24':'float32',\n# 'S_25':'float32',\n# 'S_26':'float32',\n# 'D_102':'float32',\n# 'D_103':'float32',\n# 'D_104':'float32',\n# 'D_105':'float32',\n# 'D_106':'float32',\n# 'D_107':'float32',\n# 'B_36':'float32',\n# 'B_37':'float32',\n# 'R_26':'float32',\n# 'R_27':'float32',\n# 'B_38':'category',\n# 'D_108':'float32',\n# 'D_109':'float32',\n# 'D_110':'float32',\n# 'D_111':'float32',\n# 'B_39':'float32',\n# 'D_112':'float32',\n# 'B_40':'float32',\n# 'S_27':'float32',\n# 'D_113':'float32',\n# 'D_114':'category',\n# 'D_115':'float32',\n# 'D_116':'category',\n# 'D_117':'category',\n# 'D_118':'float32',\n# 'D_119':'float32',\n# 'D_120':'category',\n# 'D_121':'float32',\n# 'D_122':'float32',\n# 'D_123':'float32',\n# 'D_124':'float32',\n# 'D_125':'float32',\n# 'D_126':'category',\n# 'D_127':'float32',\n# 'D_128':'float32',\n# 'D_129':'float32',\n# 'B_41':'float32',\n# 'B_42':'float32',\n# 'D_130':'float32',\n# 'D_131':'float32',\n# 'D_132':'float32',\n# 'D_133':'float32',\n# 'R_28':'float32',\n# 'D_134':'float32',\n# 'D_135':'float32',\n# 'D_136':'float32',\n# 'D_137':'float32',\n# 'D_138':'float32',\n# 'D_139':'float32',\n# 'D_140':'float32',\n# 'D_141':'float32',\n# 'D_142':'float32',\n# 'D_143':'float32',\n# 'D_144':'float32',\n# 'D_145':'float32'}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_data = pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', \n#                  iterator=True, chunksize=1000000,dtype = dtypes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove columns that have a large number of missing values\nfor col in train_df.columns:\n    if ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/amex-default-prediction/discussion/330347\ndel train_labels\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_defaults = train_df[train_df['target'] == 0]\ndefaults = train_df[train_df['target'] == 1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"defaults['B_6'].hist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_defaults['B_6'].hist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(train_df['B_3'], train_df[train_df['target']] == 1)\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['B_3'].describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns[-1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_col = []\nfor col in train_df.columns:\n    if not train_df[col].isnull().values.any():\n        full_col.append(col)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we can't just delete the column containg missing values bc we need the model \n# to work on the final training set too\nlen(full_col)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(train_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df.columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['B_3'].hist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Learnings: https://www.kaggle.com/competitions/amex-default-prediction/discussion/328565","metadata":{}},{"cell_type":"markdown","source":"Tabular Classification Tips and Tricks: https://www.kaggle.com/competitions/amex-default-prediction/discussion/335892","metadata":{}},{"cell_type":"markdown","source":"Times Series Classification","metadata":{}},{"cell_type":"markdown","source":"Try to develop better features than [here](https://www.kaggle.com/competitions/amex-default-prediction/discussion/335524).","metadata":{}},{"cell_type":"markdown","source":"EDA\n* https://www.kaggle.com/code/ambrosm/amex-eda-which-makes-sense - Checked\n* https://www.kaggle.com/code/azminetoushikwasi/imputation-different-techniques-with-codes - Checked\n* https://www.kaggle.com/code/cdeotte/time-series-eda - Checked\n* https://www.kaggle.com/code/raddar/the-data-has-random-uniform-noise-added - Checked\n* https://www.kaggle.com/code/datafan07/amex-where-to-begin - shows the highest correlation pairs between features; also has good histograms for all the variables\n* https://www.kaggle.com/code/ihelon/default-prediction-eda-and-modeling - good visualizations; also shows variables with high feature importance\n* https://www.kaggle.com/code/devsubhash/amex-eda-default-prediction - useful; source included at the bottom","metadata":{}},{"cell_type":"markdown","source":"Check out feature engineering discussion posts","metadata":{}},{"cell_type":"markdown","source":"Focus on D and B variables, as these are highly correlated with the target","metadata":{}},{"cell_type":"markdown","source":"Observation: \"*All train statement dates are between March of 2017 and March of 2018 (13 months), and no statement dates are missing. All test statement dates are between April of 2018 and October of 2019 (maybe the 18 months performance window refers to this).*\"\n\nObservation: \"*Now we can count how many rows (credit card statements) there are per customer. We see that 80 % of the customers have 13 statements; the other 20 % of the customers have between 1 and 12 statements.*\"\n\nSource: https://www.kaggle.com/code/ambrosm/amex-eda-which-makes-sense","metadata":{}},{"cell_type":"markdown","source":"Imputation with Time Series [Data](https://www.kaggle.com/code/parulpandey/a-guide-to-handling-missing-values-in-python).\n\nGoogle Search: Imputing Time Series Data.\n\nForward Fill and Linear interpolation seem like good imputation techniques. Advanced imputation technqiues include k-nearest neighbors imputation and multivariate feature imputation (by chained equations, aka MICE).","metadata":{}},{"cell_type":"markdown","source":"Observation: \"*As in all Kaggle competitions (and all machine learning problems, for that matter), the most important first step is to get a validation set-up that matches the test set. There's no point in spending time on feature-engineering before your validation system is trustworthy.*\"\n\n[Source](https://www.kaggle.com/competitions/home-credit-default-risk/discussion/58332).","metadata":{}},{"cell_type":"markdown","source":"**To-Do**: Plot meaningful graphs to make sense of the data.","metadata":{}},{"cell_type":"markdown","source":"My observation: There are 9 categorical delinquency variables. I wonder if null values for these are equally distributed between the training and testing data timeframes. It would be interesting if these variables had values that only existed for one set of data.\n\nOn the same note, check if there are many null values for variables in the testing data (for instance), but not in the training data. If this is the case, then it may be inappropriate to use these values.","metadata":{}},{"cell_type":"markdown","source":"14 Simple [Tips](https://www.kaggle.com/code/pavansanagapati/14-simple-tips-to-save-ram-memory-for-1-gb-dataset/notebook) to save RAM memory for 1+GB dataset.","metadata":{}},{"cell_type":"code","source":"# can write your own custom double for-loop function to manually impute the data (bfill or mean bfill)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_missing= missing_values_table(train_df)\ntrain_missing","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import missingno as msno","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getting the first 100 and then sampling 100 obviously doesn't make sense\nsampled_data = train_df.iloc[0:100,2:12].sample(100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values = missing_values_table(sampled_data)\nmissing_values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_values.index","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['D_42'].isna().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df['D_42'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 'S_3' was a float16 which needed to be converted to float for the forward fill to work\n# instead of .fillna(), you can try \n# - .interpolate(method='linear')\n# - .interpolate(option='spline')\n#sampled_data['S_3'] = sampled_data['S_3'].astype(float)\n#sampled_data['S_3_filled'] = sampled_data['S_3'].fillna(method='ffill')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampled_data.head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.fillna.html\nnumeric_columns = missing_values.index\nsampled_data[numeric_columns] = sampled_data[numeric_columns].astype(float)\nsampled_data[numeric_columns] = sampled_data[numeric_columns].fillna(method='ffill')\nsampled_data.head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Watch a theoretical video on YouTube on Linear Interpolation to develop an intuition on how it works","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualizing where the missing values are\nmsno.matrix(sampled_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sorted by S_3\nsorted_df = sampled_data.sort_values('S_3')\nmsno.matrix(sorted_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# low correlations show the data are MAR (missing at random) - there is no correlation between\n# missing values of different features\nmsno.heatmap(sorted_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"msno.dendrogram(sampled_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampled_data.iloc[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampled_data.iloc[0,:] = sampled_data.iloc[0,:].fillna(-2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampled_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.iloc[0, ['D_48, target']].corr()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/competitions/amex-default-prediction/discussion/330347\ndel old_object\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Debugging Memory Issues\nfrom memory_profiler import memory_usage\nres = memory_usage(proc())\nindex = res.index(max(res))\nwith pd.open_csv('myfile.csv') as df:\n    for dataset in df:\n        proc(dataset)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use smaller samples","metadata":{},"execution_count":null,"outputs":[]}]}