{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30840,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq\nimport os \nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T17:58:54.440457Z","iopub.execute_input":"2025-02-02T17:58:54.441003Z","iopub.status.idle":"2025-02-02T17:58:55.826743Z","shell.execute_reply.started":"2025-02-02T17:58:54.440963Z","shell.execute_reply":"2025-02-02T17:58:55.825501Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the directory where your training and test parquet files are located\ntrain_parquet_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/'\ntest_parquet_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/'\n\n# Load your training data to extract mean values\ntrain_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n\n# Initialize a DataFrame to store mean values\nmean_values_list = []","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T17:59:01.383085Z","iopub.execute_input":"2025-02-02T17:59:01.383948Z","iopub.status.idle":"2025-02-02T17:59:01.485470Z","shell.execute_reply.started":"2025-02-02T17:59:01.383912Z","shell.execute_reply":"2025-02-02T17:59:01.484135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for idx, row in train_df.iterrows():\n    file_id = row['id']\n    parquet_file_path = os.path.join(train_parquet_dir, f\"id={file_id}\", \"part-0.parquet\")\n\n    if os.path.exists(parquet_file_path): \n        data = pq.read_table(parquet_file_path).to_pandas()\n\n        feature_columns = [col for col in data.columns if col not in ['id', 'sii']]\n\n        if feature_columns: \n            mean_values = data[feature_columns].mean().values\n            mean_values_list.append(mean_values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T17:59:04.422519Z","iopub.execute_input":"2025-02-02T17:59:04.422993Z","iopub.status.idle":"2025-02-02T18:00:46.840524Z","shell.execute_reply.started":"2025-02-02T17:59:04.422958Z","shell.execute_reply":"2025-02-02T18:00:46.839113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_values_df = pd.DataFrame(mean_values_list)\nmean_values_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#sum of squared errors \nsse = []\nk_range = range(1, 11) \n\nfor k in k_range:\n    kmeans = KMeans(n_clusters=k, random_state=42)\n    kmeans.fit(mean_values_df)\n    sse.append(kmeans.inertia_)\n\nplt.figure(figsize=(10, 6))\nplt.plot(k_range, sse, marker='o')\nplt.title('Elbow Method for Optimal k')\nplt.xlabel('Number of Clusters (k)')\nplt.ylabel('Sum of Squared Errors (SSE)')\nplt.xticks(k_range)\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T18:01:38.953467Z","iopub.execute_input":"2025-02-02T18:01:38.953973Z","iopub.status.idle":"2025-02-02T18:01:39.672950Z","shell.execute_reply.started":"2025-02-02T18:01:38.953934Z","shell.execute_reply":"2025-02-02T18:01:39.671517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimal_k = 2 \nkmeans = KMeans(n_clusters=optimal_k, random_state=42)\nclusters = kmeans.fit_predict(mean_values_df)\n\nmean_values_df['Cluster'] = clusters\n\npca = PCA(n_components=2)\nmean_values_reduced = pca.fit_transform(mean_values_df.drop('Cluster', axis=1))\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T18:01:44.086702Z","iopub.execute_input":"2025-02-02T18:01:44.087104Z","iopub.status.idle":"2025-02-02T18:01:44.139095Z","shell.execute_reply.started":"2025-02-02T18:01:44.087074Z","shell.execute_reply":"2025-02-02T18:01:44.137829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_values_df_1 = pd.DataFrame(mean_values_reduced)\nmean_values_df_1.head(10)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nplt.scatter(mean_values_reduced[:, 0], mean_values_reduced[:, 1], c=mean_values_df['Cluster'], cmap='viridis', alpha=0.6)\nplt.title('K-Means Clustering Visualization')\nplt.xlabel('Principal Component 1')\nplt.ylabel('Principal Component 2')\nplt.colorbar(label='Cluster Label')\nplt.grid()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T18:01:55.688620Z","iopub.execute_input":"2025-02-02T18:01:55.689073Z","iopub.status.idle":"2025-02-02T18:01:56.077144Z","shell.execute_reply.started":"2025-02-02T18:01:55.689037Z","shell.execute_reply":"2025-02-02T18:01:56.075660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load test.csv to get ALL test IDs\ntest_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nall_test_ids = test_df['id'].tolist()  # List of all test IDs from test.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T18:15:42.661530Z","iopub.execute_input":"2025-02-02T18:15:42.662008Z","iopub.status.idle":"2025-02-02T18:15:42.675241Z","shell.execute_reply.started":"2025-02-02T18:15:42.661975Z","shell.execute_reply":"2025-02-02T18:15:42.674092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#ensure PCA is fitted before clustering \npca = PCA(n_components=2) # Set to 2 to be equal as the PCA value used in training\nmean_values_pca = pca.fit_transform(mean_values_df.drop('Cluster', axis=1)) # Fit PCA on training features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T18:15:44.016254Z","iopub.execute_input":"2025-02-02T18:15:44.016742Z","iopub.status.idle":"2025-02-02T18:15:44.030673Z","shell.execute_reply.started":"2025-02-02T18:15:44.016704Z","shell.execute_reply":"2025-02-02T18:15:44.029210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fit K-Means on the PCA-reduced training data\noptimal_k = 2 \nkmeans = KMeans(n_clusters=optimal_k, random_state=42)\nkmeans.fit(mean_values_pca)  # Now K-Means is trained on the reduced feature space","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T18:15:46.053402Z","iopub.execute_input":"2025-02-02T18:15:46.053845Z","iopub.status.idle":"2025-02-02T18:15:46.088451Z","shell.execute_reply.started":"2025-02-02T18:15:46.053813Z","shell.execute_reply":"2025-02-02T18:15:46.087396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_data = []\n\n\n# Loop through all test IDs\nfor test_file_id in all_test_ids:\n    parquet_file_path = os.path.join(test_parquet_dir, f\"id={test_file_id}\", \"part-0.parquet\")\n\n    # Check if the test parquet file exists\n    if os.path.exists(parquet_file_path):\n        # Load test parquet data\n        test_data = pq.read_table(parquet_file_path).to_pandas()\n\n        # Extract feature columns (excluding 'id')\n        feature_columns = [col for col in test_data.columns if col not in ['id']]\n\n        if feature_columns:  # Ensure there are feature columns\n            # Compute mean values for features\n            test_mean_values = test_data[feature_columns].mean().values\n\n            #Apply the same PCA transformations you did in training\n            test_mean_values_pca = pca.transform([test_mean_values])  # Transform test data to 2D\n            \n            # Predict the cluster label using K-Means\n            cluster_label = kmeans.predict(test_mean_values_pca)[0]\n\n        else: \n            cluster_label = np.nan  # Assign NaN if no valid features\n    else:\n        # If the Parquet file doesn’t exist, assigns the most frequent cluster (mode() of the training clusters).\n        cluster_label = mean_values_df['Cluster'].mode()[0]# \n\n    #Append the prediction \n    submission_data.append([test_file_id, cluster_label])  # Avoid empty rows, replace later\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T18:18:02.015786Z","iopub.execute_input":"2025-02-02T18:18:02.016265Z","iopub.status.idle":"2025-02-02T18:18:02.321293Z","shell.execute_reply.started":"2025-02-02T18:18:02.016229Z","shell.execute_reply":"2025-02-02T18:18:02.319952Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert results into a DataFrame\nsubmission_df = pd.DataFrame(submission_data, columns=['id', 'sii'])\n\n# Handle missing values (if any)\n#submission_df['sii'].fillna(submission_df['sii'].mode()[0], inplace=True)  # Replace NaNs with most common cluster\nsubmission_df['sii'] = submission_df['sii'].fillna(submission_df['sii'].mode()[0])\n# Save the submission file\nsubmission_file_path = 'submission.csv'\nsubmission_df.to_csv(submission_file_path, index=False)\n\nprint(\"✅ Submission file created:\", submission_file_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T18:19:39.750638Z","iopub.execute_input":"2025-02-02T18:19:39.751111Z","iopub.status.idle":"2025-02-02T18:19:39.763419Z","shell.execute_reply.started":"2025-02-02T18:19:39.751077Z","shell.execute_reply":"2025-02-02T18:19:39.762030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}