{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:11:38.673390Z","iopub.execute_input":"2024-11-14T10:11:38.673934Z","iopub.status.idle":"2024-11-14T10:11:38.681084Z","shell.execute_reply.started":"2024-11-14T10:11:38.673886Z","shell.execute_reply":"2024-11-14T10:11:38.679495Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## import data","metadata":{}},{"cell_type":"code","source":"train_parquet_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/'\ntest_parquet_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/'\ntrain_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:11:40.975331Z","iopub.execute_input":"2024-11-14T10:11:40.975814Z","iopub.status.idle":"2024-11-14T10:11:41.039104Z","shell.execute_reply.started":"2024-11-14T10:11:40.975759Z","shell.execute_reply":"2024-11-14T10:11:41.037780Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## EDA","metadata":{}},{"cell_type":"code","source":"train_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:11:43.708952Z","iopub.execute_input":"2024-11-14T10:11:43.709409Z","iopub.status.idle":"2024-11-14T10:11:43.762651Z","shell.execute_reply.started":"2024-11-14T10:11:43.709365Z","shell.execute_reply":"2024-11-14T10:11:43.760776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:11:45.880487Z","iopub.execute_input":"2024-11-14T10:11:45.881175Z","iopub.status.idle":"2024-11-14T10:11:45.935942Z","shell.execute_reply.started":"2024-11-14T10:11:45.881114Z","shell.execute_reply":"2024-11-14T10:11:45.934639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 儲存mean value用的dataframe\nmean_values_list = []\n# 從train資料集提取mean value\nfor idx, row in train_df.iterrows(): #使用 iterrows() 函數來遍歷 train 資料框的每一行\n    file_id = row['id'] #每次迭代中，從當前行中提取 id 欄位的值，並將其存入 file_id 變數\n    parquet_file_path = os.path.join(train_parquet_dir, f\"id={file_id}\", \"part-0.parquet\")\n\n    # 確認parquet file是否存在\n    if os.path.exists(parquet_file_path):\n        # 載入 parquet 檔案，並轉換為DataFrame\n        data = pq.read_table(parquet_file_path).to_pandas()\n\n        # 從data提取除id和sii以外的其他欄位(特徵欄位)\n        feature_columns = [col for col in data.columns if col not in ['id', 'sii']]\n\n        if feature_columns:  #計算特徵欄位的平均值\n            mean_values = data[feature_columns].mean().values\n            mean_values_list.append(mean_values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:35:25.277167Z","iopub.execute_input":"2024-11-14T10:35:25.277623Z","iopub.status.idle":"2024-11-14T10:36:07.838852Z","shell.execute_reply.started":"2024-11-14T10:35:25.277583Z","shell.execute_reply":"2024-11-14T10:36:07.837607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#把mean value list轉換成dataframe\nmean_values_df = pd.DataFrame(mean_values_list)\nmean_values_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:37:28.399388Z","iopub.execute_input":"2024-11-14T10:37:28.399836Z","iopub.status.idle":"2024-11-14T10:37:28.428896Z","shell.execute_reply.started":"2024-11-14T10:37:28.399794Z","shell.execute_reply":"2024-11-14T10:37:28.427626Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## elbow method and Silhouette Method for K-means","metadata":{}},{"cell_type":"code","source":"#使用elbow法找到最佳cluster數\nsse = []\nk_range = range(1, 10)  #可以自由調整範圍\n\nfor k in k_range:\n    kmeans = KMeans(n_clusters=k, random_state=42)\n    kmeans.fit(mean_values_df)\n    sse.append(kmeans.inertia_)\n\nplt.figure(figsize=(10, 6))\nplt.plot(k_range, sse, marker='o')\nplt.title('Elbow Method for Optimal k')\nplt.xlabel('Number of Clusters (k)')\nplt.ylabel('Sum of Squared Errors (SSE)')\nplt.xticks(k_range)\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:13:39.156426Z","iopub.execute_input":"2024-11-14T10:13:39.156905Z","iopub.status.idle":"2024-11-14T10:13:50.290085Z","shell.execute_reply.started":"2024-11-14T10:13:39.156860Z","shell.execute_reply":"2024-11-14T10:13:50.288746Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### elbow point約發生在2，創造兩個組別即可。\n有些論點表示不該使用elbow mothod 而是要採用Silhouette Method檢驗，以下是Silhouette Method","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import silhouette_score\nimport matplotlib.pyplot as plt\nfrom sklearn.cluster import KMeans\n\nsilhouette_scores = []\nk_range = range(2, 10)  # Adjust range as needed\n\nfor k in k_range:\n    kmeans = KMeans(n_clusters=k, random_state=42)\n    cluster_labels = kmeans.fit_predict(mean_values_df)\n    silhouette_avg = silhouette_score(mean_values_df, cluster_labels)\n    silhouette_scores.append(silhouette_avg)\n\nplt.figure(figsize=(10, 6))\nplt.plot(k_range, silhouette_scores, marker='o')\nplt.title('Silhouette Method for Optimal k')\nplt.xlabel('Number of Clusters (k)')\nplt.ylabel('Silhouette Score')\nplt.xticks(k_range)\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:15:46.786483Z","iopub.execute_input":"2024-11-14T10:15:46.786994Z","iopub.status.idle":"2024-11-14T10:15:56.734021Z","shell.execute_reply.started":"2024-11-14T10:15:46.786950Z","shell.execute_reply":"2024-11-14T10:15:56.732714Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"####  optimal number of clusters is usually the k that has the highest silhouette score,取k=2","metadata":{}},{"cell_type":"code","source":"#K-Means\noptimal_k = 2\nkmeans = KMeans(n_clusters=optimal_k, random_state=42)\nclusters = kmeans.fit_predict(mean_values_df)\n\n# 將群集標籤加入到 DataFrame\nmean_values_df['Cluster'] = clusters\n\n# 使用 PCA 將資料降維到 2 維\npca = PCA(n_components=2)\nmean_values_reduced = pca.fit_transform(mean_values_df.drop('Cluster', axis=1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:16:02.150047Z","iopub.execute_input":"2024-11-14T10:16:02.150510Z","iopub.status.idle":"2024-11-14T10:16:03.391633Z","shell.execute_reply.started":"2024-11-14T10:16:02.150466Z","shell.execute_reply":"2024-11-14T10:16:03.390547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nplt.scatter(mean_values_reduced[:, 0], mean_values_reduced[:, 1], c=mean_values_df['Cluster'], cmap='viridis', alpha=0.6)\nplt.title('K-Means Clustering Visualization')\nplt.xlabel('Principal Component 1')\nplt.ylabel('Principal Component 2')\nplt.colorbar(label='Cluster Label')\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:16:50.317984Z","iopub.execute_input":"2024-11-14T10:16:50.318392Z","iopub.status.idle":"2024-11-14T10:16:51.060385Z","shell.execute_reply.started":"2024-11-14T10:16:50.318353Z","shell.execute_reply":"2024-11-14T10:16:51.059329Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1. 群集 0 (purple)相對較為集中，分佈範圍小，表示此群集內的資料點可能特徵相似度較高\n2. 群集 1(yellow)則分佈較廣，且呈現往右側擴散的趨勢，表示這部分資料點在這些主成分上的值範圍較大，可能內部變異性較高。\n3. 群集 0 (purple)和群集 1 (yellow)在主成分 1 上的區別較為明顯，表明主成分 1 對這兩個群集的區分度較高。\n#### 兩個群集在主成分空間中有明顯的區隔，表示 K-Means 對這些資料的分群效果還不錯。","metadata":{}},{"cell_type":"markdown","source":"## PCA 分析\n要深入分析 K-Means 分群的結果，可以通過檢查 主成分在原始特徵上的權重 來理解不同群集之間的特徵差異。這可以幫助我們理解哪些特徵在主成分中佔據較大的權重，從而影響分群。\n#### 1.提取PCA Loadings\n在主成分分析 (PCA) 中，每個主成分是原始特徵的線性組合，且有一組權重，稱為「負載係數」。這些係數表明每個原始特徵對於該主成分的重要性。我們可以透過這些權重來分析每個主成分中的特徵貢獻。","metadata":{}},{"cell_type":"code","source":"#PCA Loadings\npca_loadings = pd.DataFrame(\n    pca.components_.T,  # 每個原始特徵在主成分上的權重\n    columns=[f'PC{i+1}' for i in range(pca.n_components_)],  # 標記主成分\n    index=mean_values_df.columns.drop('Cluster')  # 原始特徵名稱（假設 'Cluster' 不參與分析）\n)\n\n# 顯示前幾行數據以檢查每個特徵對於每個主成分的貢獻\nprint(pca_loadings.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:16:57.354349Z","iopub.execute_input":"2024-11-14T10:16:57.354827Z","iopub.status.idle":"2024-11-14T10:16:57.366357Z","shell.execute_reply.started":"2024-11-14T10:16:57.354767Z","shell.execute_reply":"2024-11-14T10:16:57.365137Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"上面表格中每一列代表一個主成分 (PC1, PC2, ...) 的權重，每一行代表一個原始特徵。較高的絕對值表示該特徵對主成分的貢獻較大。","metadata":{}},{"cell_type":"markdown","source":"#### 2.分析主要特徵對群集的影響\n在 pca_loadings 表格中，選擇每個主成分（PC1 和 PC2）上絕對值較高的特徵，對主成分的貢獻較大。","metadata":{}},{"cell_type":"code","source":"# 找出對 PC1 和 PC2 貢獻較大的特徵\ntop_features_PC1 = pca_loadings['PC1'].abs().sort_values(ascending=False).head(5)\ntop_features_PC2 = pca_loadings['PC2'].abs().sort_values(ascending=False).head(5)\n\nprint(\"Top features contributing to PC1:\")\nprint(top_features_PC1)\nprint(\"\\nTop features contributing to PC2:\")\nprint(top_features_PC2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:16:59.705177Z","iopub.execute_input":"2024-11-14T10:16:59.705620Z","iopub.status.idle":"2024-11-14T10:16:59.717158Z","shell.execute_reply.started":"2024-11-14T10:16:59.705577Z","shell.execute_reply":"2024-11-14T10:16:59.715746Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 3. 解釋各群集的特徵差異\n\n* 使用 pca_loadings 表格中的資訊，可以了解在主成分空間中佔主導地位的特徵，結合各群集在這些特徵上的均值差異可以解釋不同群集的特徵差異。\n* 比較各群集在 mean_values_df 中的特徵均值，觀察對主成分貢獻較大的特徵，以更清楚地理解群集特徵。\n","metadata":{}},{"cell_type":"code","source":"# 計算每個群集的特徵均值\ncluster_means = mean_values_df.groupby('Cluster').mean()\n\n# 視覺化對 PC1 和 PC2 貢獻較大的特徵在不同群集的均值\nplt.figure(figsize=(12, 6))\n\n# 選取對 PC1 貢獻最大的特徵\nfor feature in top_features_PC1.index:\n    plt.plot(cluster_means.index, cluster_means[feature], label=feature)\n\nplt.title('Top Features Contributing to PC1 across Clusters')\nplt.xlabel('Cluster')\nplt.ylabel('Mean Feature Value')\nplt.legend()\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:17:05.438192Z","iopub.execute_input":"2024-11-14T10:17:05.438637Z","iopub.status.idle":"2024-11-14T10:17:05.773764Z","shell.execute_reply.started":"2024-11-14T10:17:05.438591Z","shell.execute_reply":"2024-11-14T10:17:05.772377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cluster_means = mean_values_df.groupby('Cluster').mean()\nplt.figure(figsize=(12, 6))\n\n# 選取對 PC2 貢獻最大的特徵\nfor feature in top_features_PC2.index:\n    plt.plot(cluster_means.index, cluster_means[feature], label=feature)\n\nplt.title('Top Features Contributing to PC2 across Clusters')\nplt.xlabel('Cluster')\nplt.ylabel('Mean Feature Value')\nplt.legend()\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:17:07.589239Z","iopub.execute_input":"2024-11-14T10:17:07.589723Z","iopub.status.idle":"2024-11-14T10:17:07.938954Z","shell.execute_reply.started":"2024-11-14T10:17:07.589648Z","shell.execute_reply":"2024-11-14T10:17:07.937649Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## submission file ","metadata":{}},{"cell_type":"code","source":"#使用 K-Means 模型對測試資料進行分群預測，並將結果儲存到 submission_data 列表中\nsubmission_data = []\n\n#迭代指定的測試檔案 ID(隨機抽取)\nfor test_file_id in ['00115b9f', '001f3379']:\n    parquet_file_path = os.path.join(test_parquet_dir, f\"id={test_file_id}\", \"part-0.parquet\")\n    \n    # 建構 Parquet 檔案路徑\n    if os.path.exists(parquet_file_path):\n        # 讀取測試 Parquet 資料\n        test_data = pq.read_table(parquet_file_path).to_pandas()\n        \n        # 提取特徵欄位並計算均值\n        feature_columns = [col for col in test_data.columns if col not in ['id']]\n        \n        if feature_columns:  # 有feature columns才進行轉換\n            test_mean_values = test_data[feature_columns].mean().values\n            # 使用 K-Means 模型預測群集標籤\n            cluster_label = kmeans.predict([test_mean_values])[0]\n            submission_data.append([test_file_id, cluster_label])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T09:33:55.410218Z","iopub.execute_input":"2024-11-14T09:33:55.410704Z","iopub.status.idle":"2024-11-14T09:33:55.497727Z","shell.execute_reply.started":"2024-11-14T09:33:55.410643Z","shell.execute_reply":"2024-11-14T09:33:55.496519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_data = []\ntrue_labels = []  # 用來儲存真實標籤\npredicted_labels = []  # 用來儲存預測標籤\n# 迭代所有測試檔案\nfor root, dirs, files in os.walk(test_parquet_dir):\n    for file in files:\n        if file.endswith(\".parquet\"):\n            # 提取測試檔案 ID\n            test_file_id = file.split(\"=\")[-1].split(\".\")[0]\n            parquet_file_path = os.path.join(root, file)\n            \n            if os.path.exists(parquet_file_path):\n                # 讀取測試 Parquet 資料\n                test_data = pq.read_table(parquet_file_path).to_pandas()\n                \n                # 提取特徵欄位並計算均值\n                feature_columns = [col for col in test_data.columns if col not in ['id', 'true_label']]  # 假設有真實標籤列\n                if feature_columns:  # 有特徵欄位才進行預測\n                    test_mean_values = test_data[feature_columns].mean().values\n                    # 使用 K-Means 模型預測群集標籤\n                    cluster_label = kmeans.predict([test_mean_values])[0]\n                    submission_data.append([test_file_id, cluster_label])\n                    \n                    # 如果有真實標籤，計算準確率\n                    if 'true_label' in test_data.columns:\n                        true_labels.append(test_data['true_label'].iloc[0])\n                        predicted_labels.append(cluster_label)\n\n# 如果有真實標籤，計算準確率\nif true_labels and predicted_labels:\n    accuracy = accuracy_score(true_labels, predicted_labels)\n    print(f\"模型預測準確率: {accuracy:.4f}\")\nelse:\n    print(\"沒有可用的真實標籤來計算準確率\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 創建submission DataFrame\nsubmission_df = pd.DataFrame(submission_data, columns=['id', 'sii'])\n\n# 儲存submission file\nsubmission_file_path = 'submission.csv'\nsubmission_df.to_csv(submission_file_path, index=False)\n\nprint(\"Submission file created:\", submission_file_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T09:33:57.708539Z","iopub.execute_input":"2024-11-14T09:33:57.709025Z","iopub.status.idle":"2024-11-14T09:33:57.719352Z","shell.execute_reply.started":"2024-11-14T09:33:57.708979Z","shell.execute_reply":"2024-11-14T09:33:57.718047Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T09:33:59.651871Z","iopub.execute_input":"2024-11-14T09:33:59.652290Z","iopub.status.idle":"2024-11-14T09:33:59.660360Z","shell.execute_reply.started":"2024-11-14T09:33:59.652251Z","shell.execute_reply":"2024-11-14T09:33:59.658932Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### 監督式學習的 K-Means 結合方法：\n將 K-Means視為一種資料轉換或特徵增強的方式，將 K-Means群集標籤作為特徵傳遞給監督學習模型\n1. K-Means 分群：首先使用 K-Means 分群方法來對資料進行分群，然後將每個資料點的群集標籤（kmeans_labels）作為新的特徵添加到資料集中。\n2. 使用監督式學習：接著將這些新特徵與原始資料一起傳遞給監督式學習模型（隨機森林），並進行模型訓練。\n3. 準確率評估：最後使用測試資料進行預測，並計算模型的準確率。","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nimport pandas as pd\nimport os\nimport pyarrow.parquet as pq\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import KMeans","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:17:15.019432Z","iopub.execute_input":"2024-11-14T10:17:15.020270Z","iopub.status.idle":"2024-11-14T10:17:15.027262Z","shell.execute_reply.started":"2024-11-14T10:17:15.020217Z","shell.execute_reply":"2024-11-14T10:17:15.025923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = train_df['sii']\n\n# 1. 提取 mean values 並計算 K-Means 群集標籤\nmean_values_list = []\n\nfor idx, row in train_df.iterrows():\n    file_id = row['id']\n    parquet_file_path = os.path.join(train_parquet_dir, f\"id={file_id}\", \"part-0.parquet\")\n    \n    if os.path.exists(parquet_file_path):\n        # 載入 parquet 檔案\n        data = pq.read_table(parquet_file_path).to_pandas()\n        \n        # 提取特徵欄位\n        feature_columns = [col for col in data.columns if col not in ['id', 'sii']]\n        \n        if feature_columns:\n            # 計算特徵均值\n            mean_values = data[feature_columns].mean().values\n            mean_values_list.append(mean_values)\n\n# 檢查mean_values_list的長度\nprint(f\"mean_values_list的長度: {len(mean_values_list)}\")\nprint(f\"train_df的長度: {len(train_df)}\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:27:41.099369Z","iopub.execute_input":"2024-11-14T10:27:41.099842Z","iopub.status.idle":"2024-11-14T10:28:23.948642Z","shell.execute_reply.started":"2024-11-14T10:27:41.099798Z","shell.execute_reply":"2024-11-14T10:28:23.947147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 檢查mean_values_list的長度\nprint(f\"mean_values_list的長度: {len(mean_values_list)}\")\nprint(f\"train_df的長度: {len(train_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:39:49.126143Z","iopub.execute_input":"2024-11-14T10:39:49.126610Z","iopub.status.idle":"2024-11-14T10:39:49.133350Z","shell.execute_reply.started":"2024-11-14T10:39:49.126568Z","shell.execute_reply":"2024-11-14T10:39:49.132015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#初始化空列表以確保順序對齊\naligned_mean_values_list = []\n\n# 迭代 train_df 的每一行，按順序插入均值\nfor idx, row in train_df.iterrows():\n    file_id = row['id']\n    parquet_file_path = os.path.join(train_parquet_dir, f\"id={file_id}\", \"part-0.parquet\")\n\n    # 如果 parquet 文件存在，計算特徵均值，否則填充 NaN 或 0\n    if os.path.exists(parquet_file_path):\n        data = pq.read_table(parquet_file_path).to_pandas()\n        feature_columns = [col for col in data.columns if col not in ['id', 'sii']]\n        if feature_columns:\n            mean_values = data[feature_columns].mean().values\n            aligned_mean_values_list.append(mean_values)\n        else:\n            aligned_mean_values_list.append([np.nan] * len(feature_columns))\n    else:\n        # 若 parquet 不存在，添加 NaN 或 0\n        aligned_mean_values_list.append([np.nan] * len(feature_columns))\n\n# 將對齊後的 mean_values_list 轉換為 DataFrame\nmean_values_df = pd.DataFrame(aligned_mean_values_list)\nmean_values_df.columns = mean_values_df.columns.astype(str)\n\n# 確認長度一致\nprint(f\"aligned_mean_values_list 的長度: {len(aligned_mean_values_list)}\")\nprint(f\"train_df 的長度: {len(train_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:41:42.353598Z","iopub.execute_input":"2024-11-14T10:41:42.354423Z","iopub.status.idle":"2024-11-14T10:42:25.121134Z","shell.execute_reply.started":"2024-11-14T10:41:42.354372Z","shell.execute_reply":"2024-11-14T10:42:25.119678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 把mean value list轉換成DataFrame，並重置索引、確認所有列名都是字符串\nmean_values_df = pd.DataFrame(aligned_mean_values_list)\n\n# 重置索引以避免出現混合類型列\nmean_values_df.reset_index(drop=True, inplace=True)\n\n# 確保所有列名轉換為字符串\nmean_values_df.columns = mean_values_df.columns.astype(str)\n\n# 再次檢查列名稱類型\nprint(\"Column types in mean_values_df:\", mean_values_df.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:44:06.905988Z","iopub.execute_input":"2024-11-14T10:44:06.906521Z","iopub.status.idle":"2024-11-14T10:44:06.929509Z","shell.execute_reply.started":"2024-11-14T10:44:06.906474Z","shell.execute_reply":"2024-11-14T10:44:06.928015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\n\n# Use SimpleImputer to replace NaNs with the column mean\nimputer = SimpleImputer(strategy='mean')\nmean_values_df = pd.DataFrame(imputer.fit_transform(mean_values_df), columns=mean_values_df.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:46:15.188798Z","iopub.execute_input":"2024-11-14T10:46:15.189281Z","iopub.status.idle":"2024-11-14T10:46:15.202511Z","shell.execute_reply.started":"2024-11-14T10:46:15.189237Z","shell.execute_reply":"2024-11-14T10:46:15.201154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 移除非特徵列，例如 'Cluster'\nfeature_data = mean_values_df.iloc[:, :-1]  # 排除最後一列，保留前 13 列作為特徵列\n\n# 2. 使用 K-Means 分群\noptimal_k = 2\nkmeans = KMeans(n_clusters=optimal_k, random_state=42)\nclusters = kmeans.fit_predict(feature_data)\n\n# 3. 將 K-Means 群集標籤添加為新特徵\nmean_values_df['Cluster'] = clusters\n\n# 4. 使用 PCA 進行降維 (降至2維)\npca = PCA(n_components=2)\nmean_values_reduced = pca.fit_transform(mean_values_df.drop('Cluster', axis=1))\n\n# 5. 將 PCA 的結果和群集標籤合併作為特徵\nX = pd.DataFrame(mean_values_reduced, columns=['PC1', 'PC2'])\nX['Cluster'] = mean_values_df['Cluster']\n\n# 6. Target variable ('sii')\ny = train_df['sii']\n\n# 7. Remove any NaN values in y to prevent errors during training\nX, y = X[~y.isna()], y.dropna()\n\n# 6. 拆分資料集\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# 7. 訓練監督學習模型 (隨機森林作為範例)\nmodel = RandomForestClassifier(random_state=42)\nmodel.fit(X_train, y_train)\n\n# 8. 預測並計算準確率\ny_pred = model.predict(X_test)\naccuracy = accuracy_score(y_test, y_pred)\n\nprint(f\"預測準確率: {accuracy:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:51:54.020320Z","iopub.execute_input":"2024-11-14T10:51:54.020901Z","iopub.status.idle":"2024-11-14T10:51:55.604349Z","shell.execute_reply.started":"2024-11-14T10:51:54.020852Z","shell.execute_reply":"2024-11-14T10:51:55.603049Z"}},"outputs":[],"execution_count":null}]}