{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import required Packages","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T12:02:02.264528Z","iopub.execute_input":"2024-11-03T12:02:02.265688Z","iopub.status.idle":"2024-11-03T12:02:05.672216Z","shell.execute_reply.started":"2024-11-03T12:02:02.265621Z","shell.execute_reply":"2024-11-03T12:02:05.669288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the directory where your training and test parquet files are located\ntrain_parquet_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/'\ntest_parquet_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/'\n\n# Load your training data to extract mean values\ntrain_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n\n# Initialize a DataFrame to store mean values\nmean_values_list = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T12:02:05.675125Z","iopub.execute_input":"2024-11-03T12:02:05.675860Z","iopub.status.idle":"2024-11-03T12:02:05.748911Z","shell.execute_reply.started":"2024-11-03T12:02:05.675802Z","shell.execute_reply":"2024-11-03T12:02:05.747622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Process each train parquet file to extract mean values\nfor idx, row in train_df.iterrows():\n    file_id = row['id']\n    parquet_file_path = os.path.join(train_parquet_dir, f\"id={file_id}\", \"part-0.parquet\")\n    \n    # Check if the parquet file exists\n    if os.path.exists(parquet_file_path):\n        # Load parquet data for current ID\n        data = pq.read_table(parquet_file_path).to_pandas()\n        \n        # Compute mean values for feature columns\n        feature_columns = [col for col in data.columns if col not in ['id', 'sii']]\n        \n        if feature_columns:  # Only process if there are feature columns\n            mean_values = data[feature_columns].mean().values\n            mean_values_list.append(mean_values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T12:02:05.750622Z","iopub.execute_input":"2024-11-03T12:02:05.751348Z","iopub.status.idle":"2024-11-03T12:03:37.987812Z","shell.execute_reply.started":"2024-11-03T12:02:05.751294Z","shell.execute_reply":"2024-11-03T12:03:37.986281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert the mean values into a DataFrame\nmean_values_df = pd.DataFrame(mean_values_list)\nmean_values_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T12:03:37.990680Z","iopub.execute_input":"2024-11-03T12:03:37.991119Z","iopub.status.idle":"2024-11-03T12:03:38.027888Z","shell.execute_reply.started":"2024-11-03T12:03:37.991076Z","shell.execute_reply":"2024-11-03T12:03:38.026831Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot the Elbow graph to find the optimal number of clusters\nsse = []\nk_range = range(1, 11)  # You can adjust the range as needed\n\nfor k in k_range:\n    kmeans = KMeans(n_clusters=k, random_state=42)\n    kmeans.fit(mean_values_df)\n    sse.append(kmeans.inertia_)\n\nplt.figure(figsize=(10, 6))\nplt.plot(k_range, sse, marker='o')\nplt.title('Elbow Method for Optimal k')\nplt.xlabel('Number of Clusters (k)')\nplt.ylabel('Sum of Squared Errors (SSE)')\nplt.xticks(k_range)\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T12:03:38.029312Z","iopub.execute_input":"2024-11-03T12:03:38.029847Z","iopub.status.idle":"2024-11-03T12:03:49.045120Z","shell.execute_reply.started":"2024-11-03T12:03:38.029793Z","shell.execute_reply":"2024-11-03T12:03:49.043871Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Elbow point is 2, we will create only 2 groups","metadata":{}},{"cell_type":"code","source":"# After determining the optimal number of clusters from the elbow plot, apply K-Means\noptimal_k = 2\nkmeans = KMeans(n_clusters=optimal_k, random_state=42)\nclusters = kmeans.fit_predict(mean_values_df)\n\n# Add cluster labels to the mean values DataFrame\nmean_values_df['Cluster'] = clusters\n\n# Visualize clusters using PCA\npca = PCA(n_components=2)\nmean_values_reduced = pca.fit_transform(mean_values_df.drop('Cluster', axis=1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T12:03:49.046553Z","iopub.execute_input":"2024-11-03T12:03:49.046967Z","iopub.status.idle":"2024-11-03T12:03:50.370450Z","shell.execute_reply.started":"2024-11-03T12:03:49.046926Z","shell.execute_reply":"2024-11-03T12:03:50.369310Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nplt.scatter(mean_values_reduced[:, 0], mean_values_reduced[:, 1], c=mean_values_df['Cluster'], cmap='viridis', alpha=0.6)\nplt.title('K-Means Clustering Visualization')\nplt.xlabel('Principal Component 1')\nplt.ylabel('Principal Component 2')\nplt.colorbar(label='Cluster Label')\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T12:03:50.372078Z","iopub.execute_input":"2024-11-03T12:03:50.372944Z","iopub.status.idle":"2024-11-03T12:03:50.839929Z","shell.execute_reply.started":"2024-11-03T12:03:50.372887Z","shell.execute_reply":"2024-11-03T12:03:50.838376Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Creating submission","metadata":{}},{"cell_type":"code","source":"# Now, create a submission file using the test parquet files\nsubmission_data = []\n\nfor test_file_id in ['00115b9f', '001f3379']:\n    parquet_file_path = os.path.join(test_parquet_dir, f\"id={test_file_id}\", \"part-0.parquet\")\n    \n    # Check if the test parquet file exists\n    if os.path.exists(parquet_file_path):\n        # Load test parquet data for current ID\n        test_data = pq.read_table(parquet_file_path).to_pandas()\n        \n        # Compute mean values for feature columns\n        feature_columns = [col for col in test_data.columns if col not in ['id']]\n        \n        if feature_columns:  # Only process if there are feature columns\n            test_mean_values = test_data[feature_columns].mean().values\n            # Predict the cluster for the test mean values\n            cluster_label = kmeans.predict([test_mean_values])[0]\n            submission_data.append([test_file_id, cluster_label])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T12:03:50.841379Z","iopub.execute_input":"2024-11-03T12:03:50.841887Z","iopub.status.idle":"2024-11-03T12:03:51.071218Z","shell.execute_reply.started":"2024-11-03T12:03:50.841832Z","shell.execute_reply":"2024-11-03T12:03:51.070071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a submission DataFrame\nsubmission_df = pd.DataFrame(submission_data, columns=['id', 'sii'])\n\n# Save the submission file\nsubmission_file_path = 'submission.csv'\nsubmission_df.to_csv(submission_file_path, index=False)\n\nprint(\"Submission file created:\", submission_file_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T12:03:51.072699Z","iopub.execute_input":"2024-11-03T12:03:51.073115Z","iopub.status.idle":"2024-11-03T12:03:51.085680Z","shell.execute_reply.started":"2024-11-03T12:03:51.073074Z","shell.execute_reply":"2024-11-03T12:03:51.084465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}