{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pickle\nimport gc\nimport seaborn as sns\n\nimport matplotlib.pyplot as plt\nfrom scipy.stats import kurtosis,skew, norm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-04T04:20:27.881919Z","iopub.execute_input":"2022-07-04T04:20:27.882556Z","iopub.status.idle":"2022-07-04T04:20:27.887487Z","shell.execute_reply.started":"2022-07-04T04:20:27.882522Z","shell.execute_reply":"2022-07-04T04:20:27.886607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_obj(filepath):\n    with open(filepath, 'rb') as file:\n        obj = pickle.load(file)\n    return obj","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:20:27.987147Z","iopub.execute_input":"2022-07-04T04:20:27.987670Z","iopub.status.idle":"2022-07-04T04:20:27.992423Z","shell.execute_reply.started":"2022-07-04T04:20:27.987639Z","shell.execute_reply":"2022-07-04T04:20:27.991454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/amex-datasetcategorical-encoders/train_customer2id.pkl\", 'rb') as file:\n    train_customer2id = pickle.load(file)\nprint(len(train_customer2id))\n\nfeat_value_ranges = load_obj(\"../input/amex-default-value-ranges/feat_value_ranges.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:20:28.143142Z","iopub.execute_input":"2022-07-04T04:20:28.144102Z","iopub.status.idle":"2022-07-04T04:20:28.950060Z","shell.execute_reply.started":"2022-07-04T04:20:28.144063Z","shell.execute_reply":"2022-07-04T04:20:28.949060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df = pd.read_parquet(\"../input/amex-train-parquet-dataset/train_dataset.parquet\")\ntrain_label = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\ntrain_label.customer_ID=train_label.customer_ID.apply(lambda k: train_customer2id[k])\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:20:28.951784Z","iopub.execute_input":"2022-07-04T04:20:28.952488Z","iopub.status.idle":"2022-07-04T04:21:07.333255Z","shell.execute_reply.started":"2022-07-04T04:20:28.952445Z","shell.execute_reply":"2022-07-04T04:21:07.331997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_missing_value_percentages(df):\n    na_df = []\n\n    cat_features=['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', \n                  'D_126', 'D_63',  'D_64', 'D_66', 'D_68'] + ['customer_ID', 'S_2', 'target']\n    numeric_features = [colname for colname in train_df.columns if colname not in \n                        cat_features ]\n\n    for featname in numeric_features:\n        p = (df[featname].isna().sum())/len(df)\n        na_df.append({\n            'feat_name': featname,\n            'percent': p\n        })\n    na_df = pd.DataFrame.from_dict(na_df)\n    na_df = na_df.sort_values('percent')\n    return na_df","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:21:07.335233Z","iopub.execute_input":"2022-07-04T04:21:07.335596Z","iopub.status.idle":"2022-07-04T04:21:07.344872Z","shell.execute_reply.started":"2022-07-04T04:21:07.335560Z","shell.execute_reply":"2022-07-04T04:21:07.343589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nna_df = get_missing_value_percentages(train_df)\nnumeric_columns = na_df[na_df.percent<0.01].feat_name.values\nprint(\"number of numeric columns:\", len(numeric_columns))\n\nna_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:21:07.347899Z","iopub.execute_input":"2022-07-04T04:21:07.348621Z","iopub.status.idle":"2022-07-04T04:21:08.992731Z","shell.execute_reply.started":"2022-07-04T04:21:07.348546Z","shell.execute_reply":"2022-07-04T04:21:08.991558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for featname in feat_value_ranges.keys():\n    k1=feat_value_ranges[featname]['k1']\n    k2=feat_value_ranges[featname]['k2']\n    default_value = feat_value_ranges[featname]['default_value']\n\n    train_df[featname] = np.clip(train_df[featname], k1, k2)\n    train_df[featname].fillna(default_value, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:21:08.993921Z","iopub.execute_input":"2022-07-04T04:21:08.994270Z","iopub.status.idle":"2022-07-04T04:21:25.549901Z","shell.execute_reply.started":"2022-07-04T04:21:08.994239Z","shell.execute_reply":"2022-07-04T04:21:25.548864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_df.groupby('customer_ID', as_index=False)[['S_2']].count().rename(columns={'S_2': 'num_records'})\ndf = df[df.num_records==13]\ncustomer_ids = df.customer_ID.values\n\nprint(\"number of customers:\", len(customer_ids))","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:21:25.551113Z","iopub.execute_input":"2022-07-04T04:21:25.551455Z","iopub.status.idle":"2022-07-04T04:21:25.878994Z","shell.execute_reply.started":"2022-07-04T04:21:25.551425Z","shell.execute_reply":"2022-07-04T04:21:25.877992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:21:25.880138Z","iopub.execute_input":"2022-07-04T04:21:25.880450Z","iopub.status.idle":"2022-07-04T04:21:25.907188Z","shell.execute_reply.started":"2022-07-04T04:21:25.880422Z","shell.execute_reply":"2022-07-04T04:21:25.906242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_stats_by_timeseries(df, colname):\n    df=df.groupby('customer_ID')[[colname]].agg(list)\n    df = df.merge(train_label, on='customer_ID')\n    df['avg'] = df[colname].apply(np.mean)\n    df['last_record'] = df[colname].apply(lambda lst: lst[-1])\n    \n    df['local_diff'] = df['avg'] - df['last_record']\n    \n    mean0 = df[df.target==0]['local_diff'].mean()\n    median0 = np.quantile(df[df.target==0]['local_diff'], 0.5)\n    \n    mean1 = df[df.target==1]['local_diff'].mean()\n    median1 = np.quantile(df[df.target==1]['local_diff'], 0.5)\n    \n    print(\"\\t\\t\\t\", colname)\n    print()\n    print(\"\\t\\t\\tmean0 : {:.4f} | mean1 : {:.4f}\".format(mean0, mean1))\n    print(\"\\t\\t\\tmedian0 : {:.4f} | median1 : {:.4f}\".format(median0, median1))\n    \n    fig, ax=plt.subplots(1, 2, figsize=(15, 5))\n    fig.suptitle(colname)\n    ax[0].boxplot(df[df.target == 0]['local_diff'])\n    ax[1].boxplot(df[df.target == 1]['local_diff'])\n    \n    ax[0].set_title(\"non-defaulter\")\n    ax[1].set_title(\"defaulter\")\n    \n    plt.legend(loc='best')\n    plt.show()\n    \n    del df\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:24:15.148749Z","iopub.execute_input":"2022-07-04T04:24:15.149213Z","iopub.status.idle":"2022-07-04T04:24:15.161616Z","shell.execute_reply.started":"2022-07-04T04:24:15.149181Z","shell.execute_reply":"2022-07-04T04:24:15.160429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=train_df[train_df.customer_ID.isin(customer_ids)]\nfor colname in numeric_columns:\n    if colname.startswith(\"P_\"):\n        plot_stats_by_timeseries(df, colname)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:24:18.247111Z","iopub.execute_input":"2022-07-04T04:24:18.247502Z","iopub.status.idle":"2022-07-04T04:24:47.868523Z","shell.execute_reply.started":"2022-07-04T04:24:18.247470Z","shell.execute_reply":"2022-07-04T04:24:47.867435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=train_df[train_df.customer_ID.isin(customer_ids)]\nfor colname in numeric_columns:\n    if colname.startswith(\"B_\"):\n        plot_stats_by_timeseries(df, colname)","metadata":{"execution":{"iopub.status.busy":"2022-07-04T04:18:35.055679Z","iopub.execute_input":"2022-07-04T04:18:35.056275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for colname in numeric_columns:\n    if colname.startswith(\"R_\"):\n        plot_stats_by_timeseries(df, colname)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for colname in numeric_columns:\n    if colname.startswith(\"S_\"):\n        plot_stats_by_timeseries(df, colname)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for colname in numeric_columns:\n    if colname.startswith(\"D_\"):\n        plot_stats_by_timeseries(df, colname)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}