{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport cupy\nimport cudf\nimport matplotlib.pyplot as plt, gc, os\n\ntargets_dir = '../input/amex-default-prediction/train_labels.csv'\ntrain_dir   = '../input/amex-data-integer-dtypes-parquet-format/train.parquet'","metadata":{"execution":{"iopub.status.busy":"2022-06-23T09:58:25.696352Z","iopub.execute_input":"2022-06-23T09:58:25.696890Z","iopub.status.idle":"2022-06-23T09:58:28.970247Z","shell.execute_reply.started":"2022-06-23T09:58:25.696776Z","shell.execute_reply":"2022-06-23T09:58:28.969085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I was investigating the implications of N/A values in the data. I found some interesting dependencies. However, I am not really sure how to implement the knowlegde into my model.","metadata":{}},{"cell_type":"markdown","source":"Maybe this is helpful to someone. And maybe someone will be can be helpful to me:)","metadata":{}},{"cell_type":"code","source":"# Readin funtion\ndef read_file(path, usecols = None, na = True):\n    \n    print('Reading data...')\n    \n    # Distinguish for special columns if needed\n    if usecols is not None: df = cudf.read_parquet(path, columns=usecols)\n    else:                   df = cudf.read_parquet(path)\n    \n    # Reduce cus_id\n    unique_cus_ids = df.customer_ID.unique().to_pandas().values\n    assignment     = dict(zip(unique_cus_ids, list(range(len(unique_cus_ids)))))\n    df.customer_ID = df.customer_ID.to_pandas().apply(lambda x: assignment[x]).astype('int32')\n    \n    del unique_cus_ids, assignment\n    gc.collect()\n\n    # Reduce dtype for 'S_2'\n    df.S_2 = cudf.to_datetime( df.S_2 )\n    \n    # fill missing data\n    if na: df = df.fillna(NAN_VALUE)\n    \n    print('shape of data:', df.shape)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-06-23T09:58:33.637321Z","iopub.execute_input":"2022-06-23T09:58:33.637728Z","iopub.status.idle":"2022-06-23T09:58:33.649159Z","shell.execute_reply.started":"2022-06-23T09:58:33.637694Z","shell.execute_reply":"2022-06-23T09:58:33.645620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Load data for the N/A value analysis\ntest    = read_file(train_dir, na = False)\ntargets = cudf.read_csv(targets_dir)\n\ntargets['target'] = targets.target.astype('int8')\ntargets.drop('customer_ID', axis=1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-06-23T09:58:37.312969Z","iopub.execute_input":"2022-06-23T09:58:37.313438Z","iopub.status.idle":"2022-06-23T09:59:09.143754Z","shell.execute_reply.started":"2022-06-23T09:58:37.313396Z","shell.execute_reply":"2022-06-23T09:59:09.142611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build a summary table for the N/A values\nsummary = pd.DataFrame()\n\n# Which feature has how many N/A values?\nsummary['Feature']        = test.columns.values\nsummary['number_of_N/As'] = test.isna().sum().to_pandas().values\n\n# Calculate the default rate among customers with N/A values\n# 'avgs' are the default rates for by features\n# 'na_cus' is the number of customers displaying N/A values\ndef_rates_no_na_cus, def_rates_na_cus, no_na_cus, na_cus = [], [], [], []\n\nfor feature in test.columns.values:\n    \n    \n    # Get customers ids WITHOUT N/A values\n    cus  = test.loc[test[f'{feature}'].notna()].customer_ID.unique().values\n    \n    # If there are no such customers than there is no default rate among them    \n    if len(cus) == 0:\n        def_rates_no_na_cus.append(0)\n        no_na_cus.append(0)\n    else: \n        avg = targets.iloc[cus].mean().values[0]\n        def_rates_no_na_cus.append(float(avg))\n        no_na_cus.append(len(cus))\n        \n    \n    # Get customers ids WITH N/A values\n    cus1 = test.loc[test[f'{feature}'].isna()].customer_ID.unique().values\n    \n    # If there are no such customers than there is no default rate among them\n    if len(cus1) == 0:\n        def_rates_na_cus.append(0)\n        na_cus.append(0)  \n    else:\n        avg = targets.iloc[cus1].mean().values[0]\n        def_rates_na_cus.append(float(avg))\n        na_cus.append(len(cus1))\n        \n\nsummary['Cus_with_N/As']        = na_cus\nsummary['Def_rate_na_cus']      = def_rates_na_cus\nsummary['Cus_without_N/As']     = no_na_cus\nsummary['Def_rates_no_na_cus']  = def_rates_no_na_cus","metadata":{"execution":{"iopub.status.busy":"2022-06-23T10:14:29.603656Z","iopub.execute_input":"2022-06-23T10:14:29.604351Z","iopub.status.idle":"2022-06-23T10:15:06.672869Z","shell.execute_reply.started":"2022-06-23T10:14:29.604315Z","shell.execute_reply":"2022-06-23T10:15:06.670922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here you go! This is the table!","metadata":{}},{"cell_type":"code","source":"summary","metadata":{"execution":{"iopub.status.busy":"2022-06-23T10:15:27.630534Z","iopub.execute_input":"2022-06-23T10:15:27.631155Z","iopub.status.idle":"2022-06-23T10:15:27.662798Z","shell.execute_reply.started":"2022-06-23T10:15:27.631110Z","shell.execute_reply":"2022-06-23T10:15:27.661612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Avarage default rate\navg_def_rate = targets.target.mean()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that there is a number of feature where customers with N/A values display a significantly high default rates.\n\n","metadata":{}},{"cell_type":"code","source":"cols = ['Feature', 'number_of_N/As', 'Cus_with_N/As', 'Def_rate_na_cus']\nsigni_def_rate = avg_def_rate + 0.03\nsummary.sort_values('Def_rate_na_cus' ,ascending = False).loc[summary.Def_rate_na_cus > signi_def_rate, cols]","metadata":{"execution":{"iopub.status.busy":"2022-06-23T10:23:47.908497Z","iopub.execute_input":"2022-06-23T10:23:47.909664Z","iopub.status.idle":"2022-06-23T10:23:47.929948Z","shell.execute_reply.started":"2022-06-23T10:23:47.909615Z","shell.execute_reply":"2022-06-23T10:23:47.928865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['Feature', 'number_of_N/As', 'Cus_with_N/As', 'Def_rate_na_cus']\nsigni_def_rate = avg_def_rate - 0.03\nsummary.sort_values('Def_rate_na_cus' ,ascending = True).loc[summary.Def_rate_na_cus.between(0.01, signi_def_rate) , cols]","metadata":{"execution":{"iopub.status.busy":"2022-06-23T10:27:01.523663Z","iopub.execute_input":"2022-06-23T10:27:01.524266Z","iopub.status.idle":"2022-06-23T10:27:01.544215Z","shell.execute_reply.started":"2022-06-23T10:27:01.524230Z","shell.execute_reply":"2022-06-23T10:27:01.543028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This effect is even stronger when considering groups of customers with have no N/A values for some features\n","metadata":{}},{"cell_type":"code","source":"cols = ['Feature', 'number_of_N/As', 'Cus_without_N/As', 'Def_rates_no_na_cus']\nsigni_def_rate = avg_def_rate + 0.03\nsummary.sort_values('Def_rates_no_na_cus' ,ascending = False).loc[summary.Def_rates_no_na_cus > signi_def_rate, cols]","metadata":{"execution":{"iopub.status.busy":"2022-06-23T10:24:17.373213Z","iopub.execute_input":"2022-06-23T10:24:17.373571Z","iopub.status.idle":"2022-06-23T10:24:17.391226Z","shell.execute_reply.started":"2022-06-23T10:24:17.373522Z","shell.execute_reply":"2022-06-23T10:24:17.390181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['Feature', 'number_of_N/As', 'Cus_without_N/As', 'Def_rates_no_na_cus']\nsigni_def_rate = avg_def_rate - 0.03\nsummary.sort_values('Def_rates_no_na_cus' ,ascending = True).loc[summary.Def_rates_no_na_cus.between(0.01, signi_def_rate) > signi_def_rate, cols]","metadata":{"execution":{"iopub.status.busy":"2022-06-23T10:27:51.188519Z","iopub.execute_input":"2022-06-23T10:27:51.189482Z","iopub.status.idle":"2022-06-23T10:27:51.207408Z","shell.execute_reply.started":"2022-06-23T10:27:51.189441Z","shell.execute_reply":"2022-06-23T10:27:51.206214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1) Is there a way to make sense of of this?\n\n2) Currently, I am using gxboost. Using dummies to such customers is not really helpful as those groups tend to be rather small and in the algorithm the given feature will likely not be used to split the tree. Is there some way I still can implement this knowledge into my model?\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}