{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pickle\nimport gc\nimport seaborn as sns\n\nimport matplotlib.pyplot as plt\nfrom scipy.stats import kurtosis,skew, norm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-22T15:54:14.433422Z","iopub.execute_input":"2022-06-22T15:54:14.433913Z","iopub.status.idle":"2022-06-22T15:54:15.329002Z","shell.execute_reply.started":"2022-06-22T15:54:14.433826Z","shell.execute_reply":"2022-06-22T15:54:15.328238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/amex-datasetcategorical-encoders/train_customer2id.pkl\", 'rb') as file:\n    train_customer2id = pickle.load(file)\nprint(len(train_customer2id))","metadata":{"execution":{"iopub.status.busy":"2022-06-22T15:54:15.331026Z","iopub.execute_input":"2022-06-22T15:54:15.332126Z","iopub.status.idle":"2022-06-22T15:54:16.006835Z","shell.execute_reply.started":"2022-06-22T15:54:15.332077Z","shell.execute_reply":"2022-06-22T15:54:16.005713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df = pd.read_parquet(\"../input/amex-traindataset/train_dataset.parquet\")\n\ntrain_label = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\ntrain_label.customer_ID=train_label.customer_ID.apply(lambda k: train_customer2id[k])\n\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T15:54:16.008394Z","iopub.execute_input":"2022-06-22T15:54:16.009415Z","iopub.status.idle":"2022-06-22T15:54:52.206680Z","shell.execute_reply.started":"2022-06-22T15:54:16.009366Z","shell.execute_reply":"2022-06-22T15:54:52.205599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_missing_value_percentages(df):\n    na_df = []\n\n    cat_features=['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', \n                  'D_126', 'D_63',  'D_64', 'D_66', 'D_68'] + ['customer_ID', 'S_2', 'target']\n    numeric_features = [colname for colname in train_df.columns if colname not in \n                        cat_features ]\n\n    for featname in numeric_features:\n        p = (df[featname].isna().sum())/len(df)\n        na_df.append({\n            'feat_name': featname,\n            'percent': p\n        })\n    na_df = pd.DataFrame.from_dict(na_df)\n    na_df = na_df.sort_values('percent')\n    return na_df","metadata":{"execution":{"iopub.status.busy":"2022-06-22T15:54:52.209466Z","iopub.execute_input":"2022-06-22T15:54:52.209933Z","iopub.status.idle":"2022-06-22T15:54:52.218689Z","shell.execute_reply.started":"2022-06-22T15:54:52.209885Z","shell.execute_reply":"2022-06-22T15:54:52.217469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nna_df = get_missing_value_percentages(train_df)\nnumeric_columns = na_df[na_df.percent<0.01].feat_name.values\nprint(\"number of numeric columns:\", len(numeric_columns))\n\nna_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T15:54:52.220151Z","iopub.execute_input":"2022-06-22T15:54:52.221168Z","iopub.status.idle":"2022-06-22T15:54:53.943715Z","shell.execute_reply.started":"2022-06-22T15:54:52.221121Z","shell.execute_reply":"2022-06-22T15:54:53.942660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_df.groupby('customer_ID', as_index=False)[['S_2']].count().rename(columns={'S_2': 'num_records'})\ndf = df[df.num_records==13]\ncustomer_ids = df.customer_ID.values\n\nprint(\"number of customers:\", len(customer_ids))","metadata":{"execution":{"iopub.status.busy":"2022-06-22T15:54:53.945132Z","iopub.execute_input":"2022-06-22T15:54:53.945620Z","iopub.status.idle":"2022-06-22T15:54:54.216772Z","shell.execute_reply.started":"2022-06-22T15:54:53.945572Z","shell.execute_reply":"2022-06-22T15:54:54.215745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_stats_by_timeseries(df, colname):\n    df=df.groupby('customer_ID')[[colname]].agg(list)\n    df = df.merge(train_label, on='customer_ID')\n    \n    \n    series_means0 = []\n    series_means1 = []\n    \n    series_q25_0=[]\n    series_q25_1=[]\n    \n    series_q50_0=[]\n    series_q50_1=[]\n    \n    series_q75_0=[]\n    series_q75_1=[]\n\n    for k in range(13):\n        v0 = df[df.target == 0][colname].apply(lambda lst: lst[k])\n        v1 = df[df.target == 1][colname].apply(lambda lst: lst[k])\n        \n        v0 = v0[v0.isna()==False]\n        v1 = v1[v1.isna()==False]\n        \n        series_means0.append(np.mean(v0))\n        series_means1.append(np.mean(v1))\n        \n        series_q25_0.append(np.quantile(v0, 0.25))\n        series_q25_1.append(np.quantile(v1, 0.25))\n        \n        series_q50_0.append(np.quantile(v0, 0.5))\n        series_q50_1.append(np.quantile(v1, 0.5))\n        \n        series_q75_0.append(np.quantile(v0, 0.75))\n        series_q75_1.append(np.quantile(v1, 0.75))\n    \n    \n    \n    fig, ax = plt.subplots(1, 2, figsize=(15, 5), sharey=True)\n    fig.suptitle(colname)\n    \n    for k in range(13):\n        y0 = [series_q25_0[k], series_q50_0[k], series_q75_0[k] , series_means0[k]]\n        y1 = [series_q25_1[k], series_q50_1[k], series_q75_1[k] , series_means1[k]]\n        \n        \n        min_value0 = min(y0);max_value0 = max(y0)\n        min_value1 = min(y1);max_value1 = max(y1)\n        \n        ax[0].vlines(x=k, ymin=min_value0, ymax = max_value0)\n        ax[1].vlines(x=k, ymin=min_value1, ymax = max_value1)\n    \n    ax[0].set_title(\"Non Defaulter\")\n    ax[0].plot(series_means0, marker='*', color='green', label='mean')\n    ax[0].plot(series_q25_0, marker='x', color='red', label='25%')\n    ax[0].plot(series_q50_0, marker='s', color='m', label='50%')\n    ax[0].plot(series_q75_0, marker='o', color='blue', label='75%')\n    ax[0].legend(loc='upper right')\n    \n    ax[1].set_title(\"Defaulter\")\n    ax[1].plot(series_means1, marker='*', color='green', label='mean')\n    ax[1].plot(series_q25_1, marker='x', color='red', label='25%')\n    ax[1].plot(series_q50_1, marker='s', color='m', label='50%')\n    ax[1].plot(series_q75_1, marker='o', color='blue', label='75%')\n    plt.show()\n    \n    del df\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-22T15:54:54.218179Z","iopub.execute_input":"2022-06-22T15:54:54.218524Z","iopub.status.idle":"2022-06-22T15:54:54.239735Z","shell.execute_reply.started":"2022-06-22T15:54:54.218494Z","shell.execute_reply":"2022-06-22T15:54:54.238492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=train_df[train_df.customer_ID.isin(customer_ids)]\nfor colname in numeric_columns:\n    if colname.startswith(\"P_\"):\n        plot_stats_by_timeseries(df, colname)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T15:54:54.240988Z","iopub.execute_input":"2022-06-22T15:54:54.241373Z","iopub.status.idle":"2022-06-22T15:55:14.527360Z","shell.execute_reply.started":"2022-06-22T15:54:54.241342Z","shell.execute_reply":"2022-06-22T15:55:14.526439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=train_df[train_df.customer_ID.isin(customer_ids)]\nfor colname in numeric_columns:\n    if colname.startswith(\"B_\"):\n        plot_stats_by_timeseries(df, colname)","metadata":{"execution":{"iopub.status.busy":"2022-06-22T15:55:14.528557Z","iopub.execute_input":"2022-06-22T15:55:14.528852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for colname in numeric_columns:\n    if colname.startswith(\"R_\"):\n        plot_stats_by_timeseries(df, colname)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for colname in numeric_columns:\n    if colname.startswith(\"S_\"):\n        plot_stats_by_timeseries(df, colname)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for colname in numeric_columns:\n    if colname.startswith(\"D_\"):\n        plot_stats_by_timeseries(df, colname)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}