{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"bbcd69fb-fe73-48e3-86fa-8c80651e1ef5","_cell_guid":"7fa6eeae-5f9b-4b88-be31-804e74b40f85","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:36:27.159314Z","iopub.execute_input":"2022-06-23T20:36:27.160447Z","iopub.status.idle":"2022-06-23T20:36:27.206985Z","shell.execute_reply.started":"2022-06-23T20:36:27.160299Z","shell.execute_reply":"2022-06-23T20:36:27.206145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install pyspark","metadata":{"_uuid":"b5cf5b4c-2160-4e2b-8ade-d5cc8a41ac88","_cell_guid":"3cb6bc0a-48bd-4b99-b7f8-d9407d170bee","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:36:27.208357Z","iopub.execute_input":"2022-06-23T20:36:27.209188Z","iopub.status.idle":"2022-06-23T20:36:27.214942Z","shell.execute_reply.started":"2022-06-23T20:36:27.209148Z","shell.execute_reply":"2022-06-23T20:36:27.213656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nfrom pprint import pprint\n# from pyspark.sql import SparkSession, types\nfrom pandasql import sqldf\n\n# kaggle utils\nimport kaggle_utils_py as kaggle_utils\n\n# visualization tools\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\n# set the warning off\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"20cc74d2-9585-43a4-b75f-2e5a45f013cc","_cell_guid":"2a4704cb-8a5b-48a7-92e9-44aeb529e53a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:36:27.244319Z","iopub.execute_input":"2022-06-23T20:36:27.245685Z","iopub.status.idle":"2022-06-23T20:36:28.614282Z","shell.execute_reply.started":"2022-06-23T20:36:27.245612Z","shell.execute_reply":"2022-06-23T20:36:28.612941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  basic settings for me\npd.set_option('display.max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2022-06-23T20:36:28.616410Z","iopub.execute_input":"2022-06-23T20:36:28.616928Z","iopub.status.idle":"2022-06-23T20:36:28.623128Z","shell.execute_reply.started":"2022-06-23T20:36:28.616885Z","shell.execute_reply":"2022-06-23T20:36:28.621751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Read Data**","metadata":{"_uuid":"dd4f2acc-c19a-4075-b70d-62b3a40b446c","_cell_guid":"7e94462e-72c4-4534-9790-454151e23a86","trusted":true}},{"cell_type":"code","source":"%%time\n# data load\ntrain = pd.read_feather('../input/amex-default-prediction-feather/train.feather')\ntest = pd.read_feather('../input/amex-default-prediction-feather/test.feather')\ntrain_labels = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\nsub = pd.read_csv('../input/amex-default-prediction/sample_submission.csv')","metadata":{"_uuid":"262f6032-a85c-4d06-88f1-19b2fae8e6bd","_cell_guid":"115f6611-71b0-4e67-9c5f-4e2be49cfa4d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:36:28.624957Z","iopub.execute_input":"2022-06-23T20:36:28.625792Z","iopub.status.idle":"2022-06-23T20:37:22.683462Z","shell.execute_reply.started":"2022-06-23T20:36:28.625736Z","shell.execute_reply":"2022-06-23T20:37:22.682295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"shape of the data --->\", train.shape)\nprint(\"shape of the data label --->\", train_labels.shape)\nprint(\"shape of the test data --->\", test.shape)","metadata":{"_uuid":"3a82ad3f-0056-44b0-8130-9ec57784e1e1","_cell_guid":"389d65ca-d307-4156-97c3-d76e3d469b83","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:37:22.686478Z","iopub.execute_input":"2022-06-23T20:37:22.687271Z","iopub.status.idle":"2022-06-23T20:37:22.694495Z","shell.execute_reply.started":"2022-06-23T20:37:22.687214Z","shell.execute_reply":"2022-06-23T20:37:22.693399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"_uuid":"05e55aed-f61a-4da9-98e3-3138bae17f14","_cell_guid":"91795b6e-d201-4442-a1bc-d864c07e34d8","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:37:22.696328Z","iopub.execute_input":"2022-06-23T20:37:22.697064Z","iopub.status.idle":"2022-06-23T20:37:22.874316Z","shell.execute_reply.started":"2022-06-23T20:37:22.697013Z","shell.execute_reply":"2022-06-23T20:37:22.872792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head()","metadata":{"_uuid":"e66fe077-4714-4a95-9164-7806bf1879e2","_cell_guid":"9bf1b7c9-d0b4-43dd-ad9e-e54aac4f66ca","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:37:22.875980Z","iopub.execute_input":"2022-06-23T20:37:22.876517Z","iopub.status.idle":"2022-06-23T20:37:22.888343Z","shell.execute_reply.started":"2022-06-23T20:37:22.876464Z","shell.execute_reply":"2022-06-23T20:37:22.887125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"_uuid":"24365c03-b406-4833-8323-6908bb9328ca","_cell_guid":"79df5ee9-ee6a-4a52-a157-aa11209005a6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:37:22.890110Z","iopub.execute_input":"2022-06-23T20:37:22.891184Z","iopub.status.idle":"2022-06-23T20:37:23.067770Z","shell.execute_reply.started":"2022-06-23T20:37:22.891139Z","shell.execute_reply":"2022-06-23T20:37:23.066425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# variable counts \nd_feats = [c for c in train.columns if c.startswith('D_')]\ns_feats = [c for c in train.columns if c.startswith('S_')]\np_feats = [c for c in train.columns if c.startswith('P_')]\nb_feats = [c for c in train.columns if c.startswith('B_')]\nr_feats = [c for c in train.columns if c.startswith('R_')]\nprint(f'Number of Delinquency variables: {len(d_feats)}')\nprint(f'Number of Spend variables: {len(s_feats)}')\nprint(f'Number of Payment variables: {len(p_feats)}')\nprint(f'Number of Balance variables: {len(b_feats)}')\nprint(f'Number of Risk variables: {len(r_feats)}')\nprint(f'Total variable counts: {len(d_feats)+ len(s_feats)+ len(p_feats) + len(b_feats) + len(r_feats)}')","metadata":{"_uuid":"cfcbb94f-7dca-420d-af98-a532d5764c81","_cell_guid":"671ac1c5-02e4-414c-86ea-07d98d1ef2a1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:37:23.069972Z","iopub.execute_input":"2022-06-23T20:37:23.070567Z","iopub.status.idle":"2022-06-23T20:37:23.084051Z","shell.execute_reply.started":"2022-06-23T20:37:23.070513Z","shell.execute_reply":"2022-06-23T20:37:23.082618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Analysis - Customer**","metadata":{"_uuid":"3fc682c2-7962-4c30-93aa-2cde3ea8e8aa","_cell_guid":"9677a771-c7c1-4745-87b7-0b48cf0fefaf","trusted":true}},{"cell_type":"code","source":"# Customer info\nunique_customer_count = len(train.groupby(\"customer_ID\")['customer_ID'].count())\nprint(\"unique customer data in training data -->\", unique_customer_count)\nunique_customer_label_count = len(train_labels.groupby(\"customer_ID\")['customer_ID'].count())\nprint(\"unique customer data in training label data -->\", unique_customer_label_count)\nunique_customer_count_test = len(test.groupby(\"customer_ID\")['customer_ID'].count())\nprint(\"unique customer data in test data -->\", unique_customer_count_test)","metadata":{"_uuid":"16f32772-8ecd-4b00-a68b-3f8881add48e","_cell_guid":"e1f8ed63-a40b-426c-afcc-093d36e03626","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:37:23.085550Z","iopub.execute_input":"2022-06-23T20:37:23.086001Z","iopub.status.idle":"2022-06-23T20:37:30.634222Z","shell.execute_reply.started":"2022-06-23T20:37:23.085945Z","shell.execute_reply":"2022-06-23T20:37:30.632146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checking single customer data\neach_customer = train.groupby(\"customer_ID\").size()\nprint(each_customer)\nprint('Distinct count for customer data')\neach_customer.unique() # count of each customer data","metadata":{"_uuid":"6860bf89-473b-40d6-9756-69ef7b9174ac","_cell_guid":"7b25a851-bb4b-4510-bdc4-90521ee01095","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:37:30.638916Z","iopub.execute_input":"2022-06-23T20:37:30.639396Z","iopub.status.idle":"2022-06-23T20:37:32.269670Z","shell.execute_reply.started":"2022-06-23T20:37:30.639359Z","shell.execute_reply":"2022-06-23T20:37:32.268450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look at one customer data\ntrain[train[\"customer_ID\"] == \"0000099d6bd597052cdcda90ffabf56573fe9d7c79be5fbac11a8ed792feb62a\"]","metadata":{"_uuid":"ad77874b-6062-407a-b125-5c4957500757","_cell_guid":"6c186ec4-cc85-4130-a83e-13d1249bbc9b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:37:32.272152Z","iopub.execute_input":"2022-06-23T20:37:32.272665Z","iopub.status.idle":"2022-06-23T20:37:33.390601Z","shell.execute_reply.started":"2022-06-23T20:37:32.272548Z","shell.execute_reply":"2022-06-23T20:37:33.389343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count customer number for train and test \ny = train.groupby(\"customer_ID\")['customer_ID'].count().values\ny_test = test.groupby(\"customer_ID\")['customer_ID'].count().values\nprint(y, y_test)","metadata":{"_uuid":"c2cd7723-fe8b-47c0-aeef-ba8a998f4b52","_cell_guid":"7b04d3d4-b5ef-492f-b725-b9eeadb7587d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:37:33.391969Z","iopub.execute_input":"2022-06-23T20:37:33.392341Z","iopub.status.idle":"2022-06-23T20:37:40.134836Z","shell.execute_reply.started":"2022-06-23T20:37:33.392310Z","shell.execute_reply":"2022-06-23T20:37:40.133389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()\nfig.add_trace(go.Histogram(\n    y = y,\n    ybins = dict(size = 0.5),\n    marker_color= '#9900cc'))\nfig.update_layout(\n    template = \"plotly_dark\",\n    title = \"Customer profile count -- training data\",\n    yaxis_title = \"Number of months\",\n    bargap = 0.2\n)\nfig.show()\n\nfig = go.Figure()\nfig.add_trace(go.Histogram(\n    y = y_test,\n    ybins = dict(size = 0.5),\n    marker_color= '#9900cc'))\nfig.update_layout(\n    template = \"plotly_dark\",\n    title = \"Customer profile count -- test data\",\n    yaxis_title = \"Number of months\"\n)\nfig.show()\n\n# dsitribution of profile length is common between train and test data.","metadata":{"execution":{"iopub.status.busy":"2022-06-23T20:37:40.136848Z","iopub.execute_input":"2022-06-23T20:37:40.137436Z","iopub.status.idle":"2022-06-23T20:37:41.338925Z","shell.execute_reply.started":"2022-06-23T20:37:40.137382Z","shell.execute_reply":"2022-06-23T20:37:41.337504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# connection between the profile length and target output\n# match between the customer_id, count for each customer_id, and target \ncount = train.groupby(\"customer_ID\")['customer_ID'].count()\ncustomer_count_target_df = pd.DataFrame({\"customer_ID\":count.index, \"count\": count.values})\n# merge the data with the label data frame\ncustomer_count_target_df = customer_count_target_df.merge(train_labels, on='customer_ID', how='left')\ncustomer_count_target_df","metadata":{"execution":{"iopub.status.busy":"2022-06-23T20:37:41.340498Z","iopub.execute_input":"2022-06-23T20:37:41.340922Z","iopub.status.idle":"2022-06-23T20:37:44.090046Z","shell.execute_reply.started":"2022-06-23T20:37:41.340887Z","shell.execute_reply":"2022-06-23T20:37:44.088757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data = customer_count_target_df,y='count',hue='target', orient='h')\n# profile length and target doesn't seem to have a huge correlation: each profile length has about 30-50% that are target 1\n","metadata":{"execution":{"iopub.status.busy":"2022-06-23T20:37:44.091412Z","iopub.execute_input":"2022-06-23T20:37:44.091835Z","iopub.status.idle":"2022-06-23T20:37:44.555627Z","shell.execute_reply.started":"2022-06-23T20:37:44.091801Z","shell.execute_reply":"2022-06-23T20:37:44.553982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Analysis - Feature**","metadata":{"_uuid":"50a7b558-114f-427b-bf10-22c06be5ac8d","_cell_guid":"2615a085-6ef4-4940-b86b-d97806079785","trusted":true}},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-06-23T20:37:44.557246Z","iopub.execute_input":"2022-06-23T20:37:44.557638Z","iopub.status.idle":"2022-06-23T20:37:44.770794Z","shell.execute_reply.started":"2022-06-23T20:37:44.557603Z","shell.execute_reply":"2022-06-23T20:37:44.769650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# merge the train and train label\ntrain2 = train.groupby('customer_ID').tail(1).set_index('customer_ID')\ndata = train2.merge(train_labels, on='customer_ID', how='left')","metadata":{"execution":{"iopub.status.busy":"2022-06-23T20:37:44.772560Z","iopub.execute_input":"2022-06-23T20:37:44.773044Z","iopub.status.idle":"2022-06-23T20:37:48.883017Z","shell.execute_reply.started":"2022-06-23T20:37:44.772995Z","shell.execute_reply":"2022-06-23T20:37:48.881704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data","metadata":{"execution":{"iopub.status.busy":"2022-06-23T20:42:28.717111Z","iopub.execute_input":"2022-06-23T20:42:28.718095Z","iopub.status.idle":"2022-06-23T20:42:28.926182Z","shell.execute_reply.started":"2022-06-23T20:42:28.718047Z","shell.execute_reply":"2022-06-23T20:42:28.924981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Type for each column/feature \ncolumns, categorical_col, numerical_col,missing_value_df = kaggle_utils.Common_data_analysis(train, missing_value_highlight_threshold=5.0, display_df = False,only_show_missing=False)","metadata":{"_uuid":"b4b0b47c-0272-4593-8fe0-a69b7488a3a0","_cell_guid":"0dec6c21-8148-4bae-b701-9b1670909533","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T21:28:42.265092Z","iopub.execute_input":"2022-06-23T21:28:42.266108Z","iopub.status.idle":"2022-06-23T21:42:36.381322Z","shell.execute_reply.started":"2022-06-23T21:28:42.266061Z","shell.execute_reply":"2022-06-23T21:42:36.380117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# by dataset defenistion descrete columns are \n# Categorical Data: ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\ndescrete_cols=['B_30', 'B_38', 'D_63', 'D_64', 'D_66', 'D_68',\n          'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'target']\n\n# so numerical columns we need to check\nnumerical_col = [c for c in numerical_col if c not in descrete_cols]\n\ntarget_col = 'target'\n\n#all categorial columns are stored in categorical_col\ncategorical_col.extend(descrete_cols)","metadata":{"execution":{"iopub.status.busy":"2022-06-23T21:42:36.393707Z","iopub.execute_input":"2022-06-23T21:42:36.394124Z","iopub.status.idle":"2022-06-23T21:42:36.409070Z","shell.execute_reply.started":"2022-06-23T21:42:36.394087Z","shell.execute_reply":"2022-06-23T21:42:36.408041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# null value analysis\nprint(\"shape of missing value df\", missing_value_df.shape)\nmissing_value_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-23T21:42:36.411077Z","iopub.execute_input":"2022-06-23T21:42:36.411437Z","iopub.status.idle":"2022-06-23T21:42:36.436654Z","shell.execute_reply.started":"2022-06-23T21:42:36.411408Z","shell.execute_reply":"2022-06-23T21:42:36.435856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Distribution Analysis for features **","metadata":{}},{"cell_type":"code","source":"def plot_hist(data, columns, nrow, ncol, figsize, hue_value=None):\n    # find the distubution of the data.\n    fig, ax = plt.subplots(nrow,ncol, figsize=figsize)\n    col, row = ncol,nrow\n    col_count = 0\n    sns.set_style('dark')\n    for r in range(row):\n        for c in range(col):\n            if col_count >= len(columns):\n                ax[r,c].text(0.5, 0.5, \"no data\")\n            else:\n                sns.kdeplot(data=data, x=columns[col_count], hue=hue_value, ax=ax[r, c], palette=['#9900cc','#99ff99'],\n                                fill = True, hue_order=[1,0], legend = True)\n                ax[r,c].set(xlabel = columns[col_count], ylabel=(\"Density\" if c==0 else ''))\n                col_count +=1\n        # print(\"col count \", col_count)","metadata":{"execution":{"iopub.status.busy":"2022-06-23T21:42:36.437977Z","iopub.execute_input":"2022-06-23T21:42:36.438773Z","iopub.status.idle":"2022-06-23T21:42:36.452925Z","shell.execute_reply.started":"2022-06-23T21:42:36.438733Z","shell.execute_reply":"2022-06-23T21:42:36.451801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Risk variable\n# Find the distribution of risk variables\nr_feats = [c for c in r_feats if c not in descrete_cols]\nplot_hist(data, r_feats, 8, 4, (50,50),hue_value=target_col)\n#### Can't see any feature following normal distribution\n#### We can't use parameterised models -- best go for some non-parameterised models","metadata":{"execution":{"iopub.status.busy":"2022-06-23T21:42:36.454455Z","iopub.execute_input":"2022-06-23T21:42:36.455313Z","iopub.status.idle":"2022-06-23T21:43:33.509446Z","shell.execute_reply.started":"2022-06-23T21:42:36.455263Z","shell.execute_reply":"2022-06-23T21:43:33.506022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correlation with target\n#col = [c for c in data.columns if data[c].dtypes != 'object']\n\ncorr = data.corrwith(data[target_col], axis=0)\nval = [str(round(v ,2) *100) + '%' for v in corr.values]\n\nfig = go.Figure()\nfig.add_trace(go.Bar(y=corr.index, x= corr.values,\n                     orientation='h',\n                     marker_color = '#9900cc',\n                     text = val,\n                     textposition = 'outside',\n                     textfont_color = '#ffff80'))\nfig.update_layout(template = 'plotly_dark',\n                  title = \"Correlation with Target\",\n                  width = 800,\n                  height = 3000)\nfig.update_xaxes(range=[-2,2])\n\n# negative correlation top 5: P_2 -67%, B_2 -56%, B_18 -55%, B_33 -52%, D_62 -37% \n# postive correlation top 5: B_9 54%, D_55 54%, D_44 53%, D_61 53%, B_3 51%","metadata":{"execution":{"iopub.status.busy":"2022-06-23T21:53:21.815506Z","iopub.execute_input":"2022-06-23T21:53:21.816298Z","iopub.status.idle":"2022-06-23T21:53:24.370914Z","shell.execute_reply.started":"2022-06-23T21:53:21.816238Z","shell.execute_reply":"2022-06-23T21:53:24.369615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Target 0/1 distribution**","metadata":{}},{"cell_type":"code","source":"# plot the target\ncount = data[target_col].value_counts()\nprint(count)\nprint(\"percentage of not default --- >\",count[0]/data.shape[0])\nprint(\"percentage of default --->\", count[1]/data.shape[0])\nfig = go.Figure()\nfig.add_trace(go.Bar(x= ['Paid', \"Default\"],y=count.values,\n                     marker_color = ['#9900cc','#ffff80'],\n                     text = [str(round(count[0]/data.shape[0],2) * 100) + '%' , str(round(count[1]/data.shape[0], 2) * 100) + '%']))\nfig.update_layout(template = 'plotly_dark',\n                  title = \"target value distribution\",\n                  width = 500,\n                  height = 500)","metadata":{"execution":{"iopub.status.busy":"2022-06-23T22:21:57.449642Z","iopub.execute_input":"2022-06-23T22:21:57.450077Z","iopub.status.idle":"2022-06-23T22:21:57.511322Z","shell.execute_reply.started":"2022-06-23T22:21:57.450044Z","shell.execute_reply":"2022-06-23T22:21:57.510039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Appendix**","metadata":{"_uuid":"876d9eef-ab2f-44c4-8cc8-a38d8d98d513","_cell_guid":"6cebc36d-02d6-4efd-9254-fe6b77a03dac","trusted":true}},{"cell_type":"code","source":"# # Spark Session\n# # In order to use pyspark we need to create or get spark instance.\n# spark = SparkSession.builder.master(\"local[*]\").getOrCreate()","metadata":{"_uuid":"9965544f-2a50-470a-9089-efae98dd5773","_cell_guid":"f2986617-ec6e-43aa-b465-fbd2c62d4796","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:42:22.196697Z","iopub.status.idle":"2022-06-23T20:42:22.197185Z","shell.execute_reply.started":"2022-06-23T20:42:22.197010Z","shell.execute_reply":"2022-06-23T20:42:22.197029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Data Path: # need to check if we can use others generated parquet file below\n# train_data_path = '../input/amex-default-prediction/train_data.csv'\n# train_labels_path = '../input/amex-default-prediction/train_labels.csv'\n# test_data_path = '../input/amex-default-prediction/test_data.csv'\n# submission_sample_path = '../input/amex-default-prediction/sample_submission.csv'","metadata":{"_uuid":"1b454b27-90f3-4a3e-b26a-aba6c4eeb322","_cell_guid":"43ba82fc-fb95-488f-8ced-d61e81561204","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:42:22.198060Z","iopub.status.idle":"2022-06-23T20:42:22.198525Z","shell.execute_reply.started":"2022-06-23T20:42:22.198346Z","shell.execute_reply":"2022-06-23T20:42:22.198363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Load Data\n# df_train = spark.read.option(\"header\", \"true\").csv(train_data_path)\n# df_train_label = spark.read.option(\"header\", \"true\").csv(train_labels_path)\n# df_test = spark.read.option(\"header\", \"true\").csv(test_data_path)","metadata":{"_uuid":"47b7da11-0ecd-4862-a12d-ce3dbf673643","_cell_guid":"d92fd2f1-8c71-41b9-b056-e4b3d55cf44f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:42:22.199432Z","iopub.status.idle":"2022-06-23T20:42:22.199899Z","shell.execute_reply.started":"2022-06-23T20:42:22.199732Z","shell.execute_reply":"2022-06-23T20:42:22.199750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train_label.show()\n# df_train.show()","metadata":{"_uuid":"84b0aecc-00a5-47af-91ca-32f8a8739f47","_cell_guid":"74bb032d-e2d5-407c-81e1-0dc142d5240e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:42:22.200774Z","iopub.status.idle":"2022-06-23T20:42:22.201251Z","shell.execute_reply.started":"2022-06-23T20:42:22.201082Z","shell.execute_reply":"2022-06-23T20:42:22.201100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# newdf = spark.read.format(\"csv\").option(\"header\", \"true\").load(train_data_path)","metadata":{"_uuid":"8f00f09e-0c85-4aef-a317-497c977d7f0c","_cell_guid":"1a8e8f7c-fa08-4650-8c7d-a8f5d21b1553","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:42:22.202122Z","iopub.status.idle":"2022-06-23T20:42:22.202591Z","shell.execute_reply.started":"2022-06-23T20:42:22.202408Z","shell.execute_reply":"2022-06-23T20:42:22.202426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display(newdf)","metadata":{"_uuid":"1c1bd0ff-7f1f-4508-9051-d49b3514e64e","_cell_guid":"d9d268c6-1c1d-4272-a5b0-1cd4e91d399f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:42:22.203865Z","iopub.status.idle":"2022-06-23T20:42:22.204342Z","shell.execute_reply.started":"2022-06-23T20:42:22.204164Z","shell.execute_reply":"2022-06-23T20:42:22.204181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prettydf = newdf.toPandas()","metadata":{"_uuid":"697ecd0d-65ea-4436-bf3e-e476fd971c03","_cell_guid":"f9f3d55d-ab5f-4fd3-9e8a-3ca66376596e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:42:22.205211Z","iopub.status.idle":"2022-06-23T20:42:22.205696Z","shell.execute_reply.started":"2022-06-23T20:42:22.205508Z","shell.execute_reply":"2022-06-23T20:42:22.205525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Data Path: # need to check if we can use others generated parquet file below\n# train_data_path = '../input/amex-data-integer-dtypes-parquet-format/train.parquet'\n# train_labels_path = '../input/amex-default-prediction/train_labels.csv'\n# test_data_path = '../input/amex-data-integer-dtypes-parquet-format/test.parquet'\n# submission_sample_path = '../input/amex-default-prediction/sample_submission.csv'\n\n# # Load Data: \n# train_data = pd.read_parquet(train_data_path)\n# train_labels = pd.read_csv(train_labels_path)\n# test_data = pd.read_parquet(test_data_path)\n# submission = pd.read_csv(submission_sample_path)\n\n# print(train_data.shape, train_labels.shape)\n# print(test_data.shape, submission.shape)","metadata":{"_uuid":"00059273-7c89-46d9-a6b4-f01a1d24e82f","_cell_guid":"36b3bbe8-bd39-4883-b649-677d3b7bace7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:42:22.206608Z","iopub.status.idle":"2022-06-23T20:42:22.206964Z","shell.execute_reply.started":"2022-06-23T20:42:22.206797Z","shell.execute_reply":"2022-06-23T20:42:22.206813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"1f15fc5f-7a41-4e85-8d48-f792547a79f7","_cell_guid":"4ddf5e79-c950-4a99-831e-b25aa38e6fa5","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 5 features: \n# D_* = Delinquency variables - Bojun\n# S_* = Spend variables - Cecilia \n# P_* = Payment variables - Yinuo\n# B_* = Balance variables - Hanjing\n# R_* = Risk variables - Dora","metadata":{"_uuid":"0964fbad-b959-4be6-81c6-483e74d224eb","_cell_guid":"c2987b9f-3669-4d68-ba61-7c876a70f376","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:42:22.208200Z","iopub.status.idle":"2022-06-23T20:42:22.208589Z","shell.execute_reply.started":"2022-06-23T20:42:22.208397Z","shell.execute_reply":"2022-06-23T20:42:22.208414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Appendix:\n# Join Zoom Meeting\n# https://mit.zoom.us/j/3705116583","metadata":{"_uuid":"da6279f0-2a73-457d-99b6-a84c0f76b452","_cell_guid":"66a72770-7957-4ca9-9594-3d6942be3b9f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-23T20:42:22.209653Z","iopub.status.idle":"2022-06-23T20:42:22.210026Z","shell.execute_reply.started":"2022-06-23T20:42:22.209847Z","shell.execute_reply":"2022-06-23T20:42:22.209863Z"},"trusted":true},"execution_count":null,"outputs":[]}]}