{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Task. Customer Segmentation\n\n- 데이터를 정제해서, 고객별로 재정렬이 필요합니다. (aggregation)\n\n\n- row가 고객별 데이터로 묶이고 난 다음, 고객들을 유형별로 나눠봅니다.\n\n\n- 여러 가지 클러스터링 알고리즘을 사용하여, 결과를 테스트해봅니다.\n\n\n- 클러스터링을 위한 전처리부터, 평가까지 모든 항목을 하나하나 살펴보면서 데이터를 뜯어봅니다.","metadata":{}},{"cell_type":"markdown","source":"### Data Description\n\nSource : https://www.kaggle.com/c/instacart-market-basket-analysis","metadata":{}},{"cell_type":"markdown","source":"### 1. 데이터 불러오기 ","metadata":{}},{"cell_type":"code","source":"data_path = \"../input/instacart-market-basket-analysis/\"","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:02:38.045172Z","iopub.execute_input":"2022-08-10T11:02:38.045676Z","iopub.status.idle":"2022-08-10T11:02:38.058105Z","shell.execute_reply.started":"2022-08-10T11:02:38.045571Z","shell.execute_reply":"2022-08-10T11:02:38.056958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#start, end = [int(x) for x in input(\"돌려보고 싶은 클러스터 개수의 시작과 끝 범위를 입력해주세요 : \").split(\",\")]\nstart, end = 2, 20\n#cluster_model = input(\"사용할 클러스터링 모델을 입력하세요(kmeans/hac/dbscan/spectral) : \")\ncluster_model = \"kmeans\"\n#column_level = input(\"user matrix에 사용할 column을 입력하세요(department/aisle/product_name) : \")\ncolumn_level = \"department\"\nPCA_mode = False\nif PCA_mode:\n    n_components = int(input(\"PCA에 사용할 n_components 개수를 입력하세요 : \"))\n    \nquick_test = True\nK = list(range(start, end+1))","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:02:38.060682Z","iopub.execute_input":"2022-08-10T11:02:38.061864Z","iopub.status.idle":"2022-08-10T11:02:38.081784Z","shell.execute_reply.started":"2022-08-10T11:02:38.061807Z","shell.execute_reply":"2022-08-10T11:02:38.080545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Data Preparation","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\norders = pd.read_csv(data_path + \"orders.csv\")\nif quick_test: prior = pd.read_csv(data_path + \"order_products__prior.csv\")[:1000000]\nelse : prior = pd.read_csv(data_path + \"order_products__prior.csv\")\ntrain = pd.read_csv(data_path + \"order_products__train.csv\")\nproducts = pd.read_csv(data_path + \"products.csv\")\naisle = pd.read_csv(data_path + \"aisles.csv\")\ndepartment = pd.read_csv(data_path + \"departments.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:02:38.084371Z","iopub.execute_input":"2022-08-10T11:02:38.084891Z","iopub.status.idle":"2022-08-10T11:02:55.453034Z","shell.execute_reply.started":"2022-08-10T11:02:38.084819Z","shell.execute_reply":"2022-08-10T11:02:55.452098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 불러온 모든 테이블을 합칩니다.\ntemp = pd.merge(prior, products, on=[\"product_id\"])\ntemp = pd.merge(temp, orders, on=[\"order_id\"])\ntemp = pd.merge(temp, aisle, on=[\"aisle_id\"])\ndata = pd.merge(temp, department, on=[\"department_id\"])\ndel temp\ndata","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:02:55.455131Z","iopub.execute_input":"2022-08-10T11:02:55.455562Z","iopub.status.idle":"2022-08-10T11:02:57.821983Z","shell.execute_reply.started":"2022-08-10T11:02:55.455517Z","shell.execute_reply":"2022-08-10T11:02:57.820984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 거래내역에 대해서 각 물품을 얼마나 샀을까?\nif quick_test:\n    display(data.product_name.value_counts()[:10]) # top10","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:02:57.823233Z","iopub.execute_input":"2022-08-10T11:02:57.823551Z","iopub.status.idle":"2022-08-10T11:02:57.958133Z","shell.execute_reply.started":"2022-08-10T11:02:57.823519Z","shell.execute_reply":"2022-08-10T11:02:57.957090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 고객의 산 물건의 수\nif quick_test:\n    display(data.user_id.value_counts()[:10])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:02:57.959675Z","iopub.execute_input":"2022-08-10T11:02:57.960097Z","iopub.status.idle":"2022-08-10T11:02:57.999629Z","shell.execute_reply.started":"2022-08-10T11:02:57.960053Z","shell.execute_reply":"2022-08-10T11:02:57.998906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 거래내역에 포함된 소분류별 개수\nif quick_test:\n    display(data.aisle.value_counts()[:10])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:02:58.001027Z","iopub.execute_input":"2022-08-10T11:02:58.001601Z","iopub.status.idle":"2022-08-10T11:02:58.100743Z","shell.execute_reply.started":"2022-08-10T11:02:58.001543Z","shell.execute_reply":"2022-08-10T11:02:58.099387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 거래내역에 포함된 대분류별 개수\nif quick_test:\n    display(data.department.value_counts()[:10])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:02:58.103178Z","iopub.execute_input":"2022-08-10T11:02:58.103759Z","iopub.status.idle":"2022-08-10T11:02:58.194580Z","shell.execute_reply.started":"2022-08-10T11:02:58.103722Z","shell.execute_reply":"2022-08-10T11:02:58.193160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. 데이터 전처리\n\n- 데이터를 transaction 단위로 변경합니다.\n\n- 각자의 방법대로 데이터의 단위를 정해봅시다.\n\n- 결측치를 처리하고, 정규화도 진행해봅니다.\n\n- 필요하면 PCA나 SVD를 사용해도 상관없습니다.\n\n\n> User 단위로 어떤 물품을 구매했는지의 정보만 가지는 feature vector로 변환한다. e.g. pd.crosstab, CountVectorizer","metadata":{}},{"cell_type":"code","source":"columns = [\"product_name\", \"user_id\", \"aisle\", \"department\"] #data.columns\ncolumns = np.array(columns)\ncolumns = np.setdiff1d(data.columns, columns) # 차집합 구하는 함수.\ndata.drop(columns=columns, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:02:58.197429Z","iopub.execute_input":"2022-08-10T11:02:58.198009Z","iopub.status.idle":"2022-08-10T11:02:58.368322Z","shell.execute_reply.started":"2022-08-10T11:02:58.197969Z","shell.execute_reply":"2022-08-10T11:02:58.367538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if column_level == \"department\": user_matrix = pd.crosstab(data.user_id, data.department) # 21d\nelif column_level == \"aisle\" :  user_matrix = pd.crosstab(data.user_id, data.aisle) # 134d\nelse: user_matrix = pd.crosstab(data.user_id, data.product_name) #csr_matrix ## sparse matrix\nuser_matrix","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:02:58.369674Z","iopub.execute_input":"2022-08-10T11:02:58.370295Z","iopub.status.idle":"2022-08-10T11:03:02.368457Z","shell.execute_reply.started":"2022-08-10T11:02:58.370252Z","shell.execute_reply":"2022-08-10T11:03:02.367466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. 클러스터링 모델 적용하기\n\n- 사용하는 클러스터링 모델은 KMeans와 AgglomerativeClustering으로 합니다.\n\n(원하시면 DBSCAN이나 SpectralClustering을 사용해보셔도 됩니다. 단, 시간이 매우매우 오래 걸릴수 있으니 주의하세요..)\n\n**[K-Means]**\n\n- Elbow method를 이용하여 최적의 K값을 찾아보세요.\n\n\n- sparse한 특징을 가지는 데이터를 클러스터링 하기 위해서는 어떤 기법을 사용해야 할까요?\n\n\n- 클러스터링 결과를 시각화해보고, 실루엣 지수도 계산해봅시다.\n\n\n\n**[Hierarchical Clustering]**\n\n- 클러스터 개수를 4로 지정하고, linkage와 affinity를 바꿔가면서 실험해보세요.\n\n\n- 어떤 linkage와 affinity를 쓸지 고민하려면, 어떤 방법을 사용해보면 좋을까요?\n\n\n- 클러스터링 결과를 시각화해보고, 실루엣 지수도 계산해봅시다.","metadata":{}},{"cell_type":"code","source":"X = user_matrix.values\nprint(X.shape)\n\nif quick_test:\n    from sklearn.manifold import TSNE\n\n    # tSNE : 시각화용도의 차원감소 기법. (2차원으로 변환해주는 기법)\n    tsne = TSNE(n_components=2)\n    #tsne.fit()\n    #tsne.transform()\n    reduced_data = tsne.fit_transform(X)\n    reduced_data","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:03:02.369809Z","iopub.execute_input":"2022-08-10T11:03:02.370122Z","iopub.status.idle":"2022-08-10T11:12:44.555196Z","shell.execute_reply.started":"2022-08-10T11:03:02.370093Z","shell.execute_reply":"2022-08-10T11:12:44.553348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if quick_test:\n    # 206209 x 2\n    plt.figure(figsize=(12, 12))\n    #sns.scatterplot(data=reduced_data)\n    plt.scatter(reduced_data[:, 0], reduced_data[:, 1], s=5, alpha=0.3)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:12:44.557769Z","iopub.execute_input":"2022-08-10T11:12:44.558306Z","iopub.status.idle":"2022-08-10T11:12:45.001812Z","shell.execute_reply.started":"2022-08-10T11:12:44.558266Z","shell.execute_reply":"2022-08-10T11:12:45.000863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import silhouette_score\nfrom sklearn.cluster import KMeans\n#from sklearn.cluster import MiniBatchKMeans\nfrom tqdm import tqdm_notebook\n\ndef find_optimal_clusters(data, K):\n\n\n    scores = [] # initialization\n\n    for n_cluster in tqdm_notebook(K):\n        model = KMeans(n_clusters=n_cluster) # 2~10\n        #model = MiniBatchKMeans(n_clusters=n_cluster, batch_size=1024)\n        pred = model.fit_predict(data)\n\n        score = silhouette_score(data, pred)\n        scores.append(score)\n        \n    optimal_K = np.array(scores).argmax() + K[0] # K\n    best_pred = KMeans(n_clusters=optimal_K).fit_predict(data)\n    if quick_test:\n        return best_pred, scores\n    else:\n        return best_pred","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:12:45.003130Z","iopub.execute_input":"2022-08-10T11:12:45.003474Z","iopub.status.idle":"2022-08-10T11:12:45.076187Z","shell.execute_reply.started":"2022-08-10T11:12:45.003405Z","shell.execute_reply":"2022-08-10T11:12:45.075329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find optimal K\n# 1) elbow method  -> yellowbrick    # 설치 이슈.\n# from yellowbrick.cluster import elbow\n\n#     elbow()\n\n# 2) Silhouette score   # sklearn\nif quick_test:\n    best_pred, scores = find_optimal_clusters(X, K)\nelse:\n    best_pred = find_optimal_clusters(X, K)\nprint(\"Find optimal K.\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:12:45.080397Z","iopub.execute_input":"2022-08-10T11:12:45.082685Z","iopub.status.idle":"2022-08-10T11:43:54.904623Z","shell.execute_reply.started":"2022-08-10T11:12:45.081097Z","shell.execute_reply":"2022-08-10T11:43:54.903706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 클러스터 개수별 실루엣 지수를 그려주는 그래프.\nif quick_test:\n    plt.figure(figsize=(8, 4))\n    plt.title(\"Silhouette Score in range %d-%d\" % (K[0], K[-1]), fontsize=14)\n    plt.xlabel(\"Number of Clusters\")\n    plt.ylabel(\"Silhouette Score\")\n    plt.plot(K, scores)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:43:54.908479Z","iopub.execute_input":"2022-08-10T11:43:54.909103Z","iopub.status.idle":"2022-08-10T11:43:55.086390Z","shell.execute_reply.started":"2022-08-10T11:43:54.909052Z","shell.execute_reply":"2022-08-10T11:43:55.085324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if quick_test:\n    # 클러스터별 색칠 공부\n    plt.figure(figsize=(8, 8))\n    plt.scatter(reduced_data[:, 0], reduced_data[:, 1], s=10, alpha=0.3, c=best_pred)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:43:55.088078Z","iopub.execute_input":"2022-08-10T11:43:55.088777Z","iopub.status.idle":"2022-08-10T11:43:56.400868Z","shell.execute_reply.started":"2022-08-10T11:43:55.088729Z","shell.execute_reply":"2022-08-10T11:43:56.399976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if quick_test:\n    # 실루엣 계산\n    print(\"Silhouette score : %.4f\" % silhouette_score(X, best_pred)) # [-1, 1]","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:43:56.402347Z","iopub.execute_input":"2022-08-10T11:43:56.402645Z","iopub.status.idle":"2022-08-10T11:45:17.157164Z","shell.execute_reply.started":"2022-08-10T11:43:56.402616Z","shell.execute_reply":"2022-08-10T11:45:17.155974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 차원이 큰 경우엔?\n# 차원의 저주 문제를 해결하기 위해서 차원 감소 기법인 PCA를 적용해봅니다.\nif PCA_mode:\n    from sklearn.decomposition import PCA\n\n    # tsne와 같습니다.\n    # 1) n_components가 int면, 해당 차원으로 감소. # 2) n_components가 float면 해당 비율만큼 보존하는 차원으로 감소.\n    pca = PCA(n_components=n_components)\n    reduced_pca = pca.fit_transform(X)\n    print(reduced_pca.shape)\n    \n    pca_columns = [f\"PC_{n}\" for n in range(1, n_components+1)]\n\n    pca_df = pd.DataFrame(data=reduced_pca, columns=pca_columns)\n    display(pca_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:45:17.158633Z","iopub.execute_input":"2022-08-10T11:45:17.158986Z","iopub.status.idle":"2022-08-10T11:45:17.164970Z","shell.execute_reply.started":"2022-08-10T11:45:17.158954Z","shell.execute_reply":"2022-08-10T11:45:17.163854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pca된 데이터로 optimal_K를 찾아보세요.\n\nif PCA_mode:\n    # Find optimal K\n    best_pred_pca, scores_pca = find_optimal_clusters(reduced_pca, K)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:45:17.166402Z","iopub.execute_input":"2022-08-10T11:45:17.166711Z","iopub.status.idle":"2022-08-10T11:45:17.177824Z","shell.execute_reply.started":"2022-08-10T11:45:17.166680Z","shell.execute_reply":"2022-08-10T11:45:17.176795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PCA_mode:\n# 클러스터 개수별 실루엣 지수를 그려주는 그래프.\n\n    plt.figure(figsize=(8, 4))\n    plt.title(\"Silhouette Score in range %d-%d\" % (K[0], K[-1]), fontsize=14)\n    plt.xlabel(\"Number of Clusters\")\n    plt.ylabel(\"Silhouette Score\")\n    plt.plot(K, scores_pca)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:45:17.179072Z","iopub.execute_input":"2022-08-10T11:45:17.179506Z","iopub.status.idle":"2022-08-10T11:45:17.449567Z","shell.execute_reply.started":"2022-08-10T11:45:17.179474Z","shell.execute_reply":"2022-08-10T11:45:17.447616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PCA_mode:\n    # PCA를 적용한 모델에 실루엣 계산\n    print(\"Silhouette score : %.4f\" % silhouette_score(reduced_pca, best_pred_pca)) # [-1, 1]","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:45:17.450657Z","iopub.status.idle":"2022-08-10T11:45:17.451099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if cluster_model == \"hac\":\n    from sklearn.cluster import AgglomerativeClustering\n\n    model = AgglomerativeClustering(n_clusters=4, affinity=\"euclidean\", linkage=\"average\")\n    if PCA_mode:\n        pred_hac = model.fit_predict(reduced_pca)\n        print(\"Silhouette score : %.4f\" % silhouette_score(reduced_pca, pred_hac)) # [-1, 1]\n    else:\n        pred_hac = model.fit_predict(X)\n        print(\"Silhouette score : %.4f\" % silhouette_score(X, pred_hac)) # [-1, 1]","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:45:17.452314Z","iopub.status.idle":"2022-08-10T11:45:17.452734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with open(f\"result_{model}_{optimal_K}_{custom_no}.txt\", \"w\") as f:\n#     f.write(str(score))\n#     f.write(~~)\n#     ..\n#     ....","metadata":{"execution":{"iopub.status.busy":"2022-08-10T11:45:17.453537Z","iopub.status.idle":"2022-08-10T11:45:17.453983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}