{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Dependancies","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq\nimport os\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:14:46.078171Z","iopub.execute_input":"2024-11-03T09:14:46.078646Z","iopub.status.idle":"2024-11-03T09:14:46.084630Z","shell.execute_reply.started":"2024-11-03T09:14:46.078605Z","shell.execute_reply":"2024-11-03T09:14:46.083314Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load and preprocess data","metadata":{}},{"cell_type":"code","source":"# Directory containing all train parquet files\ntrain_parquet_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/'\n\n# Initialize an empty DataFrame to store mean values\nmean_values_df = pd.DataFrame(columns=['id'])\n\n# Loop over each parquet file in the directory\nfor folder_name in os.listdir(train_parquet_dir):\n    file_id = folder_name.split('=')[1]  # Extract ID from folder name\n    parquet_file_path = os.path.join(train_parquet_dir, folder_name, \"part-0.parquet\")\n    \n    try:\n        # Load parquet data and ensure it has numeric columns\n        data = pq.read_table(parquet_file_path).to_pandas()\n        numeric_columns = data.select_dtypes(include='number').columns\n        \n        if numeric_columns.empty:\n            print(f\"No numeric columns found for file ID {file_id}. Skipping.\")\n            continue\n        \n        # Compute mean of each numeric column\n        mean_values = data[numeric_columns].mean().to_frame().T  # Calculate means and convert to DataFrame\n        mean_values.insert(0, 'id', file_id)  # Add 'id' column for the file ID\n\n        # Append mean values to the main DataFrame\n        mean_values_df = pd.concat([mean_values_df, mean_values], ignore_index=True)\n        \n    except Exception as e:\n        print(f\"Failed to process {file_id}: {e}\")\n\n# Display the DataFrame with mean values for each file\nprint(\"Mean values for each file:\")\nmean_values_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:14:46.114058Z","iopub.execute_input":"2024-11-03T09:14:46.114456Z","iopub.status.idle":"2024-11-03T09:15:32.066894Z","shell.execute_reply.started":"2024-11-03T09:14:46.114420Z","shell.execute_reply":"2024-11-03T09:15:32.065800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\n\n# Merge mean values DataFrame with train DataFrame to include the 'sii' label\nlabeled_data_df = mean_values_df.merge(train_df[['id', 'sii']], on='id', how='left')\n\n# Display the labeled DataFrame\nprint(\"Data with mean values and labels:\")\nlabeled_data_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:32.068929Z","iopub.execute_input":"2024-11-03T09:15:32.069278Z","iopub.status.idle":"2024-11-03T09:15:32.130919Z","shell.execute_reply.started":"2024-11-03T09:15:32.069242Z","shell.execute_reply":"2024-11-03T09:15:32.129759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Calculate correlation matrix for the labeled data\ncorrelation_matrix = labeled_data_df.drop(columns=['id']).corr()\n\n# Set up the matplotlib figure\nplt.figure(figsize=(12, 8))\n\n# Create a heatmap\nsns.heatmap(correlation_matrix, annot=True, cmap=\"coolwarm\", fmt=\".2f\", linewidths=0.5)\n\n# Add title\nplt.title(\"Correlation Heatmap for labeled_data_df\")\n\n# Show plot\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:32.132248Z","iopub.execute_input":"2024-11-03T09:15:32.132591Z","iopub.status.idle":"2024-11-03T09:15:33.147611Z","shell.execute_reply.started":"2024-11-03T09:15:32.132556Z","shell.execute_reply":"2024-11-03T09:15:33.146466Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preparing the data for Model training","metadata":{}},{"cell_type":"code","source":"# Create DataFrame with features and labels\nX = labeled_data_df.drop(columns=['id', 'step', 'sii'])\ny = labeled_data_df['sii']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:33.150114Z","iopub.execute_input":"2024-11-03T09:15:33.150490Z","iopub.status.idle":"2024-11-03T09:15:33.156958Z","shell.execute_reply.started":"2024-11-03T09:15:33.150437Z","shell.execute_reply":"2024-11-03T09:15:33.155817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:33.158595Z","iopub.execute_input":"2024-11-03T09:15:33.159131Z","iopub.status.idle":"2024-11-03T09:15:33.183258Z","shell.execute_reply.started":"2024-11-03T09:15:33.159081Z","shell.execute_reply":"2024-11-03T09:15:33.182010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:33.184758Z","iopub.execute_input":"2024-11-03T09:15:33.185127Z","iopub.status.idle":"2024-11-03T09:15:33.195885Z","shell.execute_reply.started":"2024-11-03T09:15:33.185091Z","shell.execute_reply":"2024-11-03T09:15:33.194614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Split data into train and test sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=150)\n\nprint(X_train.shape)\nprint(X_test.shape)\nprint(y_train.shape)\nprint(y_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:33.197295Z","iopub.execute_input":"2024-11-03T09:15:33.197654Z","iopub.status.idle":"2024-11-03T09:15:33.209081Z","shell.execute_reply.started":"2024-11-03T09:15:33.197618Z","shell.execute_reply":"2024-11-03T09:15:33.207979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Normalize the data\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:33.210861Z","iopub.execute_input":"2024-11-03T09:15:33.211235Z","iopub.status.idle":"2024-11-03T09:15:33.222769Z","shell.execute_reply.started":"2024-11-03T09:15:33.211200Z","shell.execute_reply":"2024-11-03T09:15:33.221747Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Random Forest","metadata":{}},{"cell_type":"code","source":"# Initialize and train the RandomForestClassifier\nclf = RandomForestClassifier(random_state=2)\nclf.fit(X_train_scaled, y_train)\n\n# Make predictions and evaluate the model\ny_pred = clf.predict(X_test_scaled)\naccuracy = accuracy_score(y_test, y_pred)\n\nprint(f\"Model accuracy: {accuracy * 100:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:33.224115Z","iopub.execute_input":"2024-11-03T09:15:33.224557Z","iopub.status.idle":"2024-11-03T09:15:33.633740Z","shell.execute_reply.started":"2024-11-03T09:15:33.224510Z","shell.execute_reply":"2024-11-03T09:15:33.632689Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Creating Submission","metadata":{}},{"cell_type":"code","source":"# Paths to test parquet files\ntest_parquet_files = [\n    '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/id=00115b9f',\n    '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/id=001f3379'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:33.636876Z","iopub.execute_input":"2024-11-03T09:15:33.637202Z","iopub.status.idle":"2024-11-03T09:15:33.641909Z","shell.execute_reply.started":"2024-11-03T09:15:33.637168Z","shell.execute_reply":"2024-11-03T09:15:33.640762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Placeholder for test data\ntest_data = []\ntest_ids = []\n\n# Process each test parquet file to extract mean values\nfor test_file in test_parquet_files:\n    # Extract ID from the file path\n    file_id = os.path.basename(test_file).split('=')[1]\n    \n    try:\n        # Load test data\n        data = pq.read_table(test_file).to_pandas()\n        \n        # Compute mean values for feature columns\n        feature_columns = [col for col in data.columns if col not in ['id', 'step', 'sii']]\n        mean_values = data[feature_columns].mean().values\n        \n        # Store ID and features\n        test_ids.append(file_id)\n        test_data.append(mean_values)\n    \n    except:\n        pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:33.643259Z","iopub.execute_input":"2024-11-03T09:15:33.643620Z","iopub.status.idle":"2024-11-03T09:15:33.715237Z","shell.execute_reply.started":"2024-11-03T09:15:33.643585Z","shell.execute_reply":"2024-11-03T09:15:33.714077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert test data to DataFrame\nX_test_submission = pd.DataFrame(test_data, index=test_ids)\n\nX_test_submission_scaled = scaler.transform(X_test_submission)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:33.717163Z","iopub.execute_input":"2024-11-03T09:15:33.717540Z","iopub.status.idle":"2024-11-03T09:15:33.726093Z","shell.execute_reply.started":"2024-11-03T09:15:33.717499Z","shell.execute_reply":"2024-11-03T09:15:33.724922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict using the best model from hyperparameter tuning\ny_pred_submission = clf.predict(X_test_submission_scaled)\n\n# Create submission DataFrame\nsubmission_df = pd.DataFrame({\n    'id': test_ids,\n    'sii': y_pred_submission\n})\n\n# Save submission file\nsubmission_path = '/kaggle/working/submission.csv'\nsubmission_df.to_csv(submission_path, index=False)\n\nprint(f\"Submission file created at: {submission_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:15:33.727509Z","iopub.execute_input":"2024-11-03T09:15:33.727865Z","iopub.status.idle":"2024-11-03T09:15:33.751246Z","shell.execute_reply.started":"2024-11-03T09:15:33.727830Z","shell.execute_reply":"2024-11-03T09:15:33.750133Z"}},"outputs":[],"execution_count":null}]}