{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\npd.set_option(\"display.max_rows\", 500)\npd.set_option(\"display.max_columns\", 500)\npd.set_option(\"display.width\", 1000)\n\n%pylab inline\nimport seaborn as sns\n\nfrom IPython.display import set_matplotlib_formats\nset_matplotlib_formats('svg')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:16:33.065438Z","iopub.execute_input":"2022-07-08T23:16:33.065982Z","iopub.status.idle":"2022-07-08T23:16:34.231232Z","shell.execute_reply.started":"2022-07-08T23:16:33.065878Z","shell.execute_reply":"2022-07-08T23:16:34.230136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Data","metadata":{}},{"cell_type":"code","source":"path = \"../input/amex-data-integer-dtypes-parquet-format/\"\ntrain_df = pd.read_parquet(path + \"/train.parquet\")\n# test_df = pd.read_parquet(path + \"/test.parquet\")","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:16:42.670742Z","iopub.execute_input":"2022-07-08T23:16:42.671174Z","iopub.status.idle":"2022-07-08T23:17:00.328590Z","shell.execute_reply.started":"2022-07-08T23:16:42.671132Z","shell.execute_reply":"2022-07-08T23:17:00.327697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\ncustomer_target_dict = train_label.set_index(\"customer_ID\")[\"target\"].to_dict()\ntrain_df[\"label\"] = train_df[\"customer_ID\"].map(customer_target_dict)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:17:29.980238Z","iopub.execute_input":"2022-07-08T23:17:29.980692Z","iopub.status.idle":"2022-07-08T23:17:32.873739Z","shell.execute_reply.started":"2022-07-08T23:17:29.980658Z","shell.execute_reply":"2022-07-08T23:17:32.872613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_agg = train_df.groupby('customer_ID').mean().reset_index()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:17:36.975689Z","iopub.execute_input":"2022-07-08T23:17:36.976141Z","iopub.status.idle":"2022-07-08T23:18:01.019767Z","shell.execute_reply.started":"2022-07-08T23:17:36.976108Z","shell.execute_reply":"2022-07-08T23:18:01.018618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.set_index('customer_ID', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T23:18:05.670119Z","iopub.execute_input":"2022-07-08T23:18:05.670579Z","iopub.status.idle":"2022-07-08T23:18:05.679048Z","shell.execute_reply.started":"2022-07-08T23:18:05.670540Z","shell.execute_reply":"2022-07-08T23:18:05.678133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T23:18:08.819892Z","iopub.execute_input":"2022-07-08T23:18:08.821115Z","iopub.status.idle":"2022-07-08T23:18:08.932261Z","shell.execute_reply.started":"2022-07-08T23:18:08.821066Z","shell.execute_reply":"2022-07-08T23:18:08.931054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Column Type","metadata":{}},{"cell_type":"code","source":"train_df.dtypes.value_counts()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:18:11.090662Z","iopub.execute_input":"2022-07-08T23:18:11.091097Z","iopub.status.idle":"2022-07-08T23:18:11.101197Z","shell.execute_reply.started":"2022-07-08T23:18:11.091064Z","shell.execute_reply":"2022-07-08T23:18:11.100393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train_df.dtypes.unique():\n    print(col, train_df.dtypes[train_df.dtypes == col].index.to_list())\n    print('')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:18:13.380676Z","iopub.execute_input":"2022-07-08T23:18:13.381186Z","iopub.status.idle":"2022-07-08T23:18:13.393876Z","shell.execute_reply.started":"2022-07-08T23:18:13.381146Z","shell.execute_reply":"2022-07-08T23:18:13.392858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc_df = train_df.describe(include='all')\nfor col in train_df.columns:\n    desc_df.loc['unique', col] = train_df[col].nunique()\n    \ndesc_df.loc['unique'] = desc_df.loc['unique'].astype(int)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:18:15.262827Z","iopub.execute_input":"2022-07-08T23:18:15.263278Z","iopub.status.idle":"2022-07-08T23:19:43.070895Z","shell.execute_reply.started":"2022-07-08T23:18:15.263243Z","shell.execute_reply":"2022-07-08T23:19:43.069515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc_df","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:19:43.072786Z","iopub.execute_input":"2022-07-08T23:19:43.073789Z","iopub.status.idle":"2022-07-08T23:19:43.325485Z","shell.execute_reply.started":"2022-07-08T23:19:43.073751Z","shell.execute_reply":"2022-07-08T23:19:43.324661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 4))\ndesc_df.loc['unique'].plot(kind='bar')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:19:43.327034Z","iopub.execute_input":"2022-07-08T23:19:43.327570Z","iopub.status.idle":"2022-07-08T23:19:45.361064Z","shell.execute_reply.started":"2022-07-08T23:19:43.327535Z","shell.execute_reply":"2022-07-08T23:19:45.359801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"desc_df = desc_df.sort_values(by='unique', axis=1)\ndesc_df","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:19:45.362743Z","iopub.execute_input":"2022-07-08T23:19:45.363095Z","iopub.status.idle":"2022-07-08T23:19:45.615836Z","shell.execute_reply.started":"2022-07-08T23:19:45.363064Z","shell.execute_reply":"2022-07-08T23:19:45.614818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = [\n    \"B_30\", \"B_38\", \"D_114\", \"D_116\", \"D_117\",\n    \"D_120\", \"D_126\", \"D_63\", \"D_64\", \"D_66\", \"D_68\",\n]","metadata":{"execution":{"iopub.status.busy":"2022-07-08T23:19:51.890857Z","iopub.execute_input":"2022-07-08T23:19:51.891279Z","iopub.status.idle":"2022-07-08T23:19:51.896824Z","shell.execute_reply.started":"2022-07-08T23:19:51.891248Z","shell.execute_reply":"2022-07-08T23:19:51.895635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlation","metadata":{}},{"cell_type":"code","source":"corr_agg = mean_agg.corr()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:19:54.201871Z","iopub.execute_input":"2022-07-08T23:19:54.202313Z","iopub.status.idle":"2022-07-08T23:20:37.300710Z","shell.execute_reply.started":"2022-07-08T23:19:54.202280Z","shell.execute_reply":"2022-07-08T23:20:37.299472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(corr_agg)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:20:37.302975Z","iopub.execute_input":"2022-07-08T23:20:37.303467Z","iopub.status.idle":"2022-07-08T23:20:42.862466Z","shell.execute_reply.started":"2022-07-08T23:20:37.303418Z","shell.execute_reply":"2022-07-08T23:20:42.860865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 4))\ncorr_agg.loc['label'].abs().plot(kind='bar')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:20:42.864370Z","iopub.execute_input":"2022-07-08T23:20:42.865003Z","iopub.status.idle":"2022-07-08T23:20:45.007531Z","shell.execute_reply.started":"2022-07-08T23:20:42.864942Z","shell.execute_reply":"2022-07-08T23:20:45.005681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_agg.loc['label'].abs().sort_values(ascending=False).head(20)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:20:45.010617Z","iopub.execute_input":"2022-07-08T23:20:45.011946Z","iopub.status.idle":"2022-07-08T23:20:45.024946Z","shell.execute_reply.started":"2022-07-08T23:20:45.011894Z","shell.execute_reply":"2022-07-08T23:20:45.023423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Time series","metadata":{}},{"cell_type":"code","source":"sample_neg_uid = train_df[train_df['label'] == 0].index.unique()[:100]\nsample_pos_uid = train_df[train_df['label'] == 1].index.unique()[:100]","metadata":{"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:20:45.027136Z","iopub.execute_input":"2022-07-08T23:20:45.029175Z","iopub.status.idle":"2022-07-08T23:20:51.852839Z","shell.execute_reply.started":"2022-07-08T23:20:45.029088Z","shell.execute_reply":"2022-07-08T23:20:51.851902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train_df.columns[:20]:\n    if col in ['label', 'customer_ID', 'S_2']:\n        continue\n        \n    plt.figure(figsize=(10, 4))\n    \n    for uid in sample_neg_uid:\n        if not isinstance(train_df.loc[uid], pd.core.frame.DataFrame):\n            continue\n        \n        train_df.loc[uid][col].plot(c='r')\n        plt.xticks([])\n\n    for uid in sample_pos_uid:\n        if not isinstance(train_df.loc[uid], pd.core.frame.DataFrame):\n            continue\n        \n        train_df.loc[uid][col].plot(c='b')\n        plt.xticks([])\n\n    plt.legend(['target = 0', 'target = 1'])\n    plt.title(col)\n","metadata":{"scrolled":true,"tags":[],"execution":{"iopub.status.busy":"2022-07-08T23:21:35.020265Z","iopub.execute_input":"2022-07-08T23:21:35.020771Z","iopub.status.idle":"2022-07-08T23:22:16.408182Z","shell.execute_reply.started":"2022-07-08T23:21:35.020708Z","shell.execute_reply":"2022-07-08T23:22:16.406927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}