{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"bbcd69fb-fe73-48e3-86fa-8c80651e1ef5","_cell_guid":"7fa6eeae-5f9b-4b88-be31-804e74b40f85","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-25T14:35:37.514553Z","iopub.execute_input":"2022-06-25T14:35:37.515400Z","iopub.status.idle":"2022-06-25T14:35:37.560816Z","shell.execute_reply.started":"2022-06-25T14:35:37.515306Z","shell.execute_reply":"2022-06-25T14:35:37.559524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install kaggle_utils_py","metadata":{"_uuid":"b5cf5b4c-2160-4e2b-8ade-d5cc8a41ac88","_cell_guid":"3cb6bc0a-48bd-4b99-b7f8-d9407d170bee","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-25T14:35:37.563437Z","iopub.execute_input":"2022-06-25T14:35:37.564146Z","iopub.status.idle":"2022-06-25T14:35:37.568719Z","shell.execute_reply.started":"2022-06-25T14:35:37.564106Z","shell.execute_reply":"2022-06-25T14:35:37.567809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nfrom pprint import pprint\n# from pyspark.sql import SparkSession, types\nfrom pandasql import sqldf\n\n# # kaggle utils\nimport kaggle_utils_py as kaggle_utils\n\n# visualization tools\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\n# set the warning off\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"20cc74d2-9585-43a4-b75f-2e5a45f013cc","_cell_guid":"2a4704cb-8a5b-48a7-92e9-44aeb529e53a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-25T14:35:37.570364Z","iopub.execute_input":"2022-06-25T14:35:37.571191Z","iopub.status.idle":"2022-06-25T14:35:38.181144Z","shell.execute_reply.started":"2022-06-25T14:35:37.571151Z","shell.execute_reply":"2022-06-25T14:35:38.180318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  basic settings for me\npd.set_option('display.max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T14:35:38.184350Z","iopub.execute_input":"2022-06-25T14:35:38.185024Z","iopub.status.idle":"2022-06-25T14:35:38.188979Z","shell.execute_reply.started":"2022-06-25T14:35:38.184994Z","shell.execute_reply":"2022-06-25T14:35:38.188216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Read Data**","metadata":{"_uuid":"dd4f2acc-c19a-4075-b70d-62b3a40b446c","_cell_guid":"7e94462e-72c4-4534-9790-454151e23a86","trusted":true}},{"cell_type":"code","source":"%%time\n# data load\ntrain = pd.read_feather('../input/amex-default-prediction-feather/train.feather')\ntest = pd.read_feather('../input/amex-default-prediction-feather/test.feather')\ntrain_labels = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\nsub = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')","metadata":{"_uuid":"262f6032-a85c-4d06-88f1-19b2fae8e6bd","_cell_guid":"115f6611-71b0-4e67-9c5f-4e2be49cfa4d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-25T14:35:38.190284Z","iopub.execute_input":"2022-06-25T14:35:38.191319Z","iopub.status.idle":"2022-06-25T14:36:41.015258Z","shell.execute_reply.started":"2022-06-25T14:35:38.191275Z","shell.execute_reply":"2022-06-25T14:36:41.014407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"shape of the data --->\", train.shape)\nprint(\"shape of the data label --->\", train_labels.shape)\nprint(\"shape of the test data --->\", test.shape)","metadata":{"_uuid":"3a82ad3f-0056-44b0-8130-9ec57784e1e1","_cell_guid":"389d65ca-d307-4156-97c3-d76e3d469b83","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-25T14:36:41.016701Z","iopub.execute_input":"2022-06-25T14:36:41.017277Z","iopub.status.idle":"2022-06-25T14:36:41.025263Z","shell.execute_reply.started":"2022-06-25T14:36:41.017240Z","shell.execute_reply":"2022-06-25T14:36:41.024417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"_uuid":"05e55aed-f61a-4da9-98e3-3138bae17f14","_cell_guid":"91795b6e-d201-4442-a1bc-d864c07e34d8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-25T14:36:41.026727Z","iopub.execute_input":"2022-06-25T14:36:41.027139Z","iopub.status.idle":"2022-06-25T14:36:41.166975Z","shell.execute_reply.started":"2022-06-25T14:36:41.027101Z","shell.execute_reply":"2022-06-25T14:36:41.166171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head()","metadata":{"_uuid":"e66fe077-4714-4a95-9164-7806bf1879e2","_cell_guid":"9bf1b7c9-d0b4-43dd-ad9e-e54aac4f66ca","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-25T14:36:41.168348Z","iopub.execute_input":"2022-06-25T14:36:41.169440Z","iopub.status.idle":"2022-06-25T14:36:41.179185Z","shell.execute_reply.started":"2022-06-25T14:36:41.169401Z","shell.execute_reply":"2022-06-25T14:36:41.178185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"_uuid":"24365c03-b406-4833-8323-6908bb9328ca","_cell_guid":"79df5ee9-ee6a-4a52-a157-aa11209005a6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-25T14:36:41.180781Z","iopub.execute_input":"2022-06-25T14:36:41.181179Z","iopub.status.idle":"2022-06-25T14:36:41.316442Z","shell.execute_reply.started":"2022-06-25T14:36:41.181141Z","shell.execute_reply":"2022-06-25T14:36:41.315648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# variable counts \nd_feats = [c for c in train.columns if c.startswith('D_')]\ns_feats = [c for c in train.columns if c.startswith('S_')]\np_feats = [c for c in train.columns if c.startswith('P_')]\nb_feats = [c for c in train.columns if c.startswith('B_')]\nr_feats = [c for c in train.columns if c.startswith('R_')]\nprint(f'Number of Delinquency variables: {len(d_feats)}')\nprint(f'Number of Spend variables: {len(s_feats)}')\nprint(f'Number of Payment variables: {len(p_feats)}')\nprint(f'Number of Balance variables: {len(b_feats)}')\nprint(f'Number of Risk variables: {len(r_feats)}')\nprint(f'Total variable counts: {len(d_feats)+ len(s_feats)+ len(p_feats) + len(b_feats) + len(r_feats)}')","metadata":{"_uuid":"cfcbb94f-7dca-420d-af98-a532d5764c81","_cell_guid":"671ac1c5-02e4-414c-86ea-07d98d1ef2a1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-25T14:36:41.319152Z","iopub.execute_input":"2022-06-25T14:36:41.319862Z","iopub.status.idle":"2022-06-25T14:36:41.328643Z","shell.execute_reply.started":"2022-06-25T14:36:41.319824Z","shell.execute_reply":"2022-06-25T14:36:41.327517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data Analysis - Payment","metadata":{}},{"cell_type":"code","source":"def process_and_feature_engineer(df, col):\n    # INSPIRED BY\n    # https://www.kaggle.com/code/huseyincot/amex-agg-data-how-it-created\n\n    df = df.groupby(\"customer_ID\")[[col]].agg(['min', 'max', 'last', 'count'])\n    df.columns = ['_'.join(x) for x in df.columns]\n    print('shape after engineering', df.shape )\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-06-25T14:36:41.330135Z","iopub.execute_input":"2022-06-25T14:36:41.330531Z","iopub.status.idle":"2022-06-25T14:36:41.337864Z","shell.execute_reply.started":"2022-06-25T14:36:41.330493Z","shell.execute_reply":"2022-06-25T14:36:41.336884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install cuda\n# !pip install cudf\n","metadata":{"execution":{"iopub.status.busy":"2022-06-25T14:36:41.339422Z","iopub.execute_input":"2022-06-25T14:36:41.339829Z","iopub.status.idle":"2022-06-25T14:36:41.345728Z","shell.execute_reply.started":"2022-06-25T14:36:41.339791Z","shell.execute_reply":"2022-06-25T14:36:41.344788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cudf # for GPU Lib\ndef add_targets(train):\n    targets = cudf.read_csv('../input/amex-default-prediction/train_labels.csv')\n    targets['customer_ID'] = targets['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    targets = targets.set_index('customer_ID')\n    train = train.merge(targets, left_index=True, right_index=True, how='left')\n    del targets\n    return train","metadata":{"execution":{"iopub.status.busy":"2022-06-25T14:36:41.347362Z","iopub.execute_input":"2022-06-25T14:36:41.347763Z","iopub.status.idle":"2022-06-25T14:36:44.159784Z","shell.execute_reply.started":"2022-06-25T14:36:41.347712Z","shell.execute_reply":"2022-06-25T14:36:44.158952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NAN_VALUE = -127 # will fit in int8\n\ndef read_file(path = '', usecols = None):\n    # LOAD DATAFRAME\n    if usecols is not None: df = cudf.read_parquet(path, columns=usecols)\n    else: df = cudf.read_parquet(path)\n    # REDUCE DTYPE FOR CUSTOMER AND DATE\n    df['customer_ID'] = df['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    year = cudf.to_numeric(df.S_2.str[:4])\n    month = cudf.to_numeric(df.S_2.str[5:7])\n    df.S_2 = year.mul(12).add(month).sub(24207).astype('int8')\n    # FILL NAN\n    print(\"NAN Count:\",df['P_2'].isnull().sum(axis = 0),df['P_3'].isnull().sum(axis = 0),df['P_4'].isnull().sum(axis = 0))\n    df = df.fillna(NAN_VALUE) \n    print('shape of data:', df.shape)\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-06-25T14:36:44.162997Z","iopub.execute_input":"2022-06-25T14:36:44.164043Z","iopub.status.idle":"2022-06-25T14:36:44.172559Z","shell.execute_reply.started":"2022-06-25T14:36:44.163997Z","shell.execute_reply":"2022-06-25T14:36:44.171798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P_2\nTRAIN_PATH = '../input/amex-data-integer-dtypes-parquet-format/train.parquet'\ntrain_base = read_file(path = TRAIN_PATH)\nfor col in [\"P_2\", \"P_3\", \"P_4\"]:\n    df = process_and_feature_engineer(train_base,col)\n    df = add_targets(df)\n    df = df.to_pandas()\n    df = df.sort_index()\n    df = df.reset_index()\n    display(df)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T14:36:44.173911Z","iopub.execute_input":"2022-06-25T14:36:44.174992Z","iopub.status.idle":"2022-06-25T14:37:05.811544Z","shell.execute_reply.started":"2022-06-25T14:36:44.174942Z","shell.execute_reply":"2022-06-25T14:37:05.810323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Too many 0s in P_4, is there any Correlation between P_4 Count and target?\ndf = train_base[train_base[\"P_4\"]!=0]\ncol = \"P_4\"\ndf = process_and_feature_engineer(df,col)\ndf = add_targets(df)\ndf = df.to_pandas()\ndf = df.sort_index()\ndf = df.reset_index()\ndisplay(df)","metadata":{"execution":{"iopub.status.busy":"2022-06-25T14:37:05.813212Z","iopub.execute_input":"2022-06-25T14:37:05.813606Z","iopub.status.idle":"2022-06-25T14:37:05.982794Z","shell.execute_reply.started":"2022-06-25T14:37:05.813567Z","shell.execute_reply":"2022-06-25T14:37:05.981672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Validation for guess\nfrom scipy.stats import f_oneway\n \n# Running the one-way anova test between target and P_4_count\n# Assumption(H0) is that target and P_4_count are NOT correlated\ndfLists=df.groupby('target')['P_4_count'].apply(list)\n \n# Performing the ANOVA test\n# We accept the Assumption(H0) only when P-Value &gt; 0.05\nAnovaResults = f_oneway(*dfLists)\nprint('P-Value for Anova is: ', AnovaResults[1])\nif AnovaResults[1] < 0.05:\n    print(\"reject H0, they have a Correlation\")","metadata":{"execution":{"iopub.status.busy":"2022-06-25T14:37:05.984324Z","iopub.execute_input":"2022-06-25T14:37:05.984714Z","iopub.status.idle":"2022-06-25T14:37:06.029301Z","shell.execute_reply.started":"2022-06-25T14:37:05.984676Z","shell.execute_reply":"2022-06-25T14:37:06.028398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nsns.scatterplot(data=df, x='P_4_count', y='target')\n#有关系, 但是关系没那么大","metadata":{"execution":{"iopub.status.busy":"2022-06-25T14:37:06.030792Z","iopub.execute_input":"2022-06-25T14:37:06.031836Z","iopub.status.idle":"2022-06-25T14:37:06.442430Z","shell.execute_reply.started":"2022-06-25T14:37:06.031790Z","shell.execute_reply":"2022-06-25T14:37:06.441504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look at one customer data\ntrain[train[\"customer_ID\"] == \"0000099d6bd597052cdcda90ffabf56573fe9d7c79be5fbac11a8ed792feb62a\"][[\"S_2\",\"P_2\",\"P_3\",\"P_4\"]]","metadata":{"execution":{"iopub.status.busy":"2022-06-25T14:37:06.443907Z","iopub.execute_input":"2022-06-25T14:37:06.444388Z","iopub.status.idle":"2022-06-25T14:37:07.509815Z","shell.execute_reply.started":"2022-06-25T14:37:06.444346Z","shell.execute_reply":"2022-06-25T14:37:07.508840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Questions waiting for meeting\n* Note that the negative class has been subsampled for this dataset at 5%, and thus receives a 20x weighting in the scoring metric.\n-> What we should do for this comments","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}