{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# UMAP / Hdbscan exploration of Data","metadata":{}},{"cell_type":"code","source":"!mkdir -p /tmp/pip/cache/\n!cp ../input/hdbscan0827-whl/hdbscan-0.8.27-cp37-cp37m-linux_x86_64.whl /tmp/pip/cache/\n!pip install --no-index --find-links /tmp/pip/cache/ hdbscan","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:10:55.186579Z","iopub.execute_input":"2022-05-28T16:10:55.187339Z","iopub.status.idle":"2022-05-28T16:11:07.696800Z","shell.execute_reply.started":"2022-05-28T16:10:55.187239Z","shell.execute_reply":"2022-05-28T16:11:07.695503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom sklearn.preprocessing import StandardScaler\nimport umap\nimport hdbscan\nimport matplotlib.pyplot as plt\n\nDEBUG = True","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-28T16:47:46.777061Z","iopub.execute_input":"2022-05-28T16:47:46.778631Z","iopub.status.idle":"2022-05-28T16:48:01.326957Z","shell.execute_reply.started":"2022-05-28T16:47:46.778429Z","shell.execute_reply":"2022-05-28T16:48:01.325770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_data = pd.read_pickle('../input/amex-pickle-train-test/amex_train_data.pkl')\ntrain_data = train_data.drop_duplicates(subset=['customer_ID'], keep='last')\nif DEBUG:\n    train_data = train_data.sample(frac=0.2)\n    \ndate_min = train_data.S_2.min()   \ntrain_data.S_2 = (train_data.S_2 - date_min).dt.days","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:48:01.328696Z","iopub.execute_input":"2022-05-28T16:48:01.329352Z","iopub.status.idle":"2022-05-28T16:48:33.402123Z","shell.execute_reply.started":"2022-05-28T16:48:01.329314Z","shell.execute_reply":"2022-05-28T16:48:33.400546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntest_data = pd.read_pickle('../input/amex-pickle-train-test/amex_test_data.pkl')\ntest_data = test_data.drop_duplicates(subset=['customer_ID'], keep='last')\n\nif DEBUG:\n    test_data = test_data.sample(frac=0.2)\n    \ntest_data.S_2 = (test_data.S_2 - date_min).dt.days","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:50:48.711623Z","iopub.execute_input":"2022-05-28T16:50:48.712913Z","iopub.status.idle":"2022-05-28T16:51:59.755860Z","shell.execute_reply.started":"2022-05-28T16:50:48.712855Z","shell.execute_reply":"2022-05-28T16:51:59.754561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_nums = train_data.columns[train_data.dtypes=='float16']\ncol_cat = [c for c in train_data.columns if c not in col_nums]","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:52:12.693321Z","iopub.execute_input":"2022-05-28T16:52:12.694451Z","iopub.status.idle":"2022-05-28T16:52:12.702962Z","shell.execute_reply.started":"2022-05-28T16:52:12.694341Z","shell.execute_reply":"2022-05-28T16:52:12.701809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nscaler = StandardScaler()\ntrain_scaled = pd.DataFrame(scaler.fit_transform(train_data[col_nums]),columns=col_nums).fillna(0)\ntest_scaled = pd.DataFrame(scaler.transform(test_data[col_nums]),columns=col_nums).fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:52:23.885297Z","iopub.execute_input":"2022-05-28T16:52:23.885784Z","iopub.status.idle":"2022-05-28T16:52:25.660441Z","shell.execute_reply.started":"2022-05-28T16:52:23.885743Z","shell.execute_reply":"2022-05-28T16:52:25.658828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nreducer = umap.UMAP(random_state=42)\nembedding_train = reducer.fit_transform(train_scaled)\nembedding_test = reducer.transform(test_scaled)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:54:53.082321Z","iopub.execute_input":"2022-05-28T16:54:53.082770Z","iopub.status.idle":"2022-05-28T16:59:39.357451Z","shell.execute_reply.started":"2022-05-28T16:54:53.082737Z","shell.execute_reply":"2022-05-28T16:59:39.355891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nclusterer_train = hdbscan.HDBSCAN(prediction_data=True, min_cluster_size = 200 if DEBUG else 1000).fit(embedding_train)\nu_train, counts_train = np.unique(clusterer_train.labels_, return_counts=True)\n\nclusterer_test = hdbscan.HDBSCAN(prediction_data=True, min_cluster_size = 200 if DEBUG else 1000).fit(embedding_test)\nu_test, counts_test = np.unique(clusterer_test.labels_, return_counts=True)\n\nprint(u_train)\nprint(counts_train)\n\nprint(u_test)\nprint(counts_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:01:24.741787Z","iopub.execute_input":"2022-05-28T17:01:24.742296Z","iopub.status.idle":"2022-05-28T17:01:45.528798Z","shell.execute_reply.started":"2022-05-28T17:01:24.742260Z","shell.execute_reply":"2022-05-28T17:01:45.527588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.scatter(embedding_train[:, 0], embedding_train[:, 1], s=5, c=clusterer_train.labels_, edgecolors='none', cmap='jet');","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:02:38.664312Z","iopub.execute_input":"2022-05-28T17:02:38.665023Z","iopub.status.idle":"2022-05-28T17:02:39.238493Z","shell.execute_reply.started":"2022-05-28T17:02:38.664976Z","shell.execute_reply":"2022-05-28T17:02:39.237164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.scatter(embedding_test[:, 0], embedding_test[:, 1], s=5, c=clusterer_test.labels_, edgecolors='none', cmap='jet');","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:02:40.926677Z","iopub.execute_input":"2022-05-28T17:02:40.927155Z","iopub.status.idle":"2022-05-28T17:02:41.441716Z","shell.execute_reply.started":"2022-05-28T17:02:40.927120Z","shell.execute_reply":"2022-05-28T17:02:41.440658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Colors are changing (hdbscan give another order of clusters) but it seems that data distribution doesn't change that much between train and test. ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.scatter(embedding_train[:, 0], embedding_train[:, 1], s=5, c=train_data.target, edgecolors='none', cmap='jet');\nplt.colorbar();","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:08:43.324797Z","iopub.execute_input":"2022-05-28T17:08:43.325420Z","iopub.status.idle":"2022-05-28T17:08:43.916759Z","shell.execute_reply.started":"2022-05-28T17:08:43.325351Z","shell.execute_reply":"2022-05-28T17:08:43.915459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Default rate by cluster","metadata":{}},{"cell_type":"code","source":"DR_by_cluster = pd.DataFrame({'cluster':clusterer_train.labels_, 'target':train_data.target}).groupby('cluster').mean()\nDR_map = np.array([DR_by_cluster.loc[c].values[0] for c in clusterer_train.labels_])\n\nplt.figure(figsize=(10, 8))\nplt.scatter(embedding_train[:, 0], embedding_train[:, 1], s=5, c=DR_map, edgecolors='none', cmap='jet');\nplt.colorbar();","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:09:50.603215Z","iopub.execute_input":"2022-05-28T17:09:50.603890Z","iopub.status.idle":"2022-05-28T17:09:56.898403Z","shell.execute_reply.started":"2022-05-28T17:09:50.603836Z","shell.execute_reply":"2022-05-28T17:09:56.897056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Numerical","metadata":{}},{"cell_type":"markdown","source":"Main feature negatively correlate with default:","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.scatter(embedding_train[:, 0], embedding_train[:, 1], s=5, c=train_data['P_2'], edgecolors='none', cmap='jet'); #better coloring for integers ?\nplt.colorbar();\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:12:19.477050Z","iopub.execute_input":"2022-05-28T17:12:19.477611Z","iopub.status.idle":"2022-05-28T17:12:20.079695Z","shell.execute_reply.started":"2022-05-28T17:12:19.477569Z","shell.execute_reply":"2022-05-28T17:12:20.078516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"color by last date in test... relatively uniform repartition.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.scatter(embedding_test[:, 0], embedding_test[:, 1], s=5, c=test_data['S_2'], edgecolors='none', cmap='jet'); #better coloring for integers ?\nplt.colorbar();\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:13:15.269175Z","iopub.execute_input":"2022-05-28T17:13:15.270406Z","iopub.status.idle":"2022-05-28T17:13:15.851827Z","shell.execute_reply.started":"2022-05-28T17:13:15.270322Z","shell.execute_reply":"2022-05-28T17:13:15.850433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Categoricals","metadata":{}},{"cell_type":"code","source":"col_cat = ['D_63',\n 'D_64',\n 'D_66',\n 'D_68',\n 'B_30',\n 'B_38',\n 'D_114',\n 'D_116',\n 'D_117',\n 'D_120',\n 'D_126']","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:14:10.554588Z","iopub.execute_input":"2022-05-28T17:14:10.555098Z","iopub.status.idle":"2022-05-28T17:14:10.562619Z","shell.execute_reply.started":"2022-05-28T17:14:10.555061Z","shell.execute_reply":"2022-05-28T17:14:10.561560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for c in col_cat:\n    print(c)\n    plt.figure(figsize=(10, 8))\n    train_data[c].cat.categories = np.arange(len(train_data[c].cat.categories))\n    plt.scatter(embedding_train[:, 0], embedding_train[:, 1], s=5, c=train_data[c], edgecolors='none', cmap='jet'); #better coloring for integers ?\n    plt.colorbar();\n    plt.show();","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:14:23.639394Z","iopub.execute_input":"2022-05-28T17:14:23.640417Z","iopub.status.idle":"2022-05-28T17:14:29.988933Z","shell.execute_reply.started":"2022-05-28T17:14:23.640316Z","shell.execute_reply":"2022-05-28T17:14:29.987748Z"},"trusted":true},"execution_count":null,"outputs":[]}]}