{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import depedancies","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport pyarrow.parquet as pq\nimport os\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:56:26.180386Z","iopub.execute_input":"2024-11-03T09:56:26.181039Z","iopub.status.idle":"2024-11-03T09:56:29.479899Z","shell.execute_reply.started":"2024-11-03T09:56:26.180988Z","shell.execute_reply":"2024-11-03T09:56:29.478654Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Loading and Preprocessing","metadata":{}},{"cell_type":"code","source":"# Define the directory where your parquet files are located\nparquet_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet'\n\n# Load your train DataFrame (ensure you have this already)\ntrain_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:56:29.482076Z","iopub.execute_input":"2024-11-03T09:56:29.482645Z","iopub.status.idle":"2024-11-03T09:56:29.583808Z","shell.execute_reply.started":"2024-11-03T09:56:29.482600Z","shell.execute_reply":"2024-11-03T09:56:29.582546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize a DataFrame to store mean values and labels\ndata_by_id = {}\nlabel_list = []\n\n# Process each parquet file to extract mean values\nfor idx, row in train_df.iterrows():\n    file_id = row['id']\n    parquet_file_path = os.path.join(parquet_dir, f\"id={file_id}\", \"part-0.parquet\")\n    \n    # Check if the file exists\n    if not os.path.exists(parquet_file_path):\n        pass\n        continue\n    \n    try:\n        # Load parquet data for current ID\n        data = pq.read_table(parquet_file_path).to_pandas()\n        \n        # Check if data is loaded properly\n        if data.empty:\n            print(f\"No data in file: {parquet_file_path}\")\n            continue\n        \n        # Compute mean values for feature columns (exclude 'id' and 'sii')\n        feature_columns = [col for col in data.columns if col not in ['id', 'sii']]\n        \n        # Check if feature columns exist in the data before calculating means\n        if feature_columns:\n            mean_values = data[feature_columns].mean().values\n            \n            # Store mean values and label\n            data_by_id[file_id] = mean_values\n            label_list.append(row['sii'])\n    \n    except:\n        pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:56:29.585904Z","iopub.execute_input":"2024-11-03T09:56:29.586460Z","iopub.status.idle":"2024-11-03T09:58:07.811884Z","shell.execute_reply.started":"2024-11-03T09:56:29.586403Z","shell.execute_reply":"2024-11-03T09:58:07.810476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert the mean values and labels into a DataFrame\nmean_values_df = pd.DataFrame.from_dict(data_by_id, orient='index')\nmean_values_df['sii'] = label_list\nmean_values_df.reset_index(inplace=True)\nmean_values_df.rename(columns={'index': 'id'}, inplace=True)\nmean_values_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:58:07.814678Z","iopub.execute_input":"2024-11-03T09:58:07.815719Z","iopub.status.idle":"2024-11-03T09:58:07.862601Z","shell.execute_reply.started":"2024-11-03T09:58:07.815651Z","shell.execute_reply":"2024-11-03T09:58:07.861241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare data for training\nX = mean_values_df.drop(columns=['id', 'sii'])\ny = mean_values_df['sii']\n\n# Split the data into training and testing sets\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:59:02.842614Z","iopub.execute_input":"2024-11-03T09:59:02.843158Z","iopub.status.idle":"2024-11-03T09:59:02.856554Z","shell.execute_reply.started":"2024-11-03T09:59:02.843114Z","shell.execute_reply":"2024-11-03T09:59:02.855279Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Initialize the Random Forest classifier\nrf_classifier = RandomForestClassifier(n_estimators=100, random_state=2)\n\n# Train the classifier\nrf_classifier.fit(X_train, y_train)\n\n# Predict on the test set\ny_pred = rf_classifier.predict(X_test)\n\n# Display the classification report\nprint(\"Classification Report:\")\nprint(classification_report(y_test, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T09:59:17.803627Z","iopub.execute_input":"2024-11-03T09:59:17.804218Z","iopub.status.idle":"2024-11-03T09:59:18.270520Z","shell.execute_reply.started":"2024-11-03T09:59:17.804172Z","shell.execute_reply":"2024-11-03T09:59:18.269147Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Creating Submission","metadata":{}},{"cell_type":"code","source":"# Define the directory where your test parquet files are located\ntest_parquet_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/'\n\n# Define the list of test file IDs\ntest_ids = ['00115b9f', '001f3379']\n\n# Initialize a DataFrame to store test mean values\ntest_data_by_id = {}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T10:02:20.829259Z","iopub.execute_input":"2024-11-03T10:02:20.829777Z","iopub.status.idle":"2024-11-03T10:02:20.836117Z","shell.execute_reply.started":"2024-11-03T10:02:20.829730Z","shell.execute_reply":"2024-11-03T10:02:20.834714Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Process each test parquet file to extract mean values\nfor file_id in test_ids:\n    parquet_file_path = os.path.join(test_parquet_dir, f\"id={file_id}\", \"part-0.parquet\")\n    \n    # Check if the file exists\n    if not os.path.exists(parquet_file_path):\n        print(f\"Test file not found: {parquet_file_path}\")\n        continue\n    \n    try:\n        # Load parquet data for current ID\n        data = pq.read_table(parquet_file_path).to_pandas()\n        \n        # Check if data is loaded properly\n        if data.empty:\n            pass\n            continue\n        \n        # Compute mean values for feature columns (exclude 'id')\n        feature_columns = [col for col in data.columns if col not in ['id']]\n        \n        # Check if feature columns exist in the data before calculating means\n        if feature_columns:\n            mean_values = data[feature_columns].mean().values\n            \n            # Store mean values for submission\n            test_data_by_id[file_id] = mean_values\n    \n    except Exception as e:\n        pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T10:02:53.126022Z","iopub.execute_input":"2024-11-03T10:02:53.127254Z","iopub.status.idle":"2024-11-03T10:02:53.289803Z","shell.execute_reply.started":"2024-11-03T10:02:53.127180Z","shell.execute_reply":"2024-11-03T10:02:53.288729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert the test mean values into a DataFrame\ntest_mean_values_df = pd.DataFrame.from_dict(test_data_by_id, orient='index')\ntest_mean_values_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T10:03:11.448863Z","iopub.execute_input":"2024-11-03T10:03:11.449353Z","iopub.status.idle":"2024-11-03T10:03:11.473691Z","shell.execute_reply.started":"2024-11-03T10:03:11.449306Z","shell.execute_reply":"2024-11-03T10:03:11.471861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensure the model is trained only with columns that exist in the test set\ncommon_columns = test_mean_values_df.columns.intersection(X_train.columns)\n\n# Make predictions on the test data using only the common features\nif not common_columns.empty:\n    submission_predictions = rf_classifier.predict(test_mean_values_df[common_columns])\nelse:\n    print(\"No common features to make predictions.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T10:03:59.862491Z","iopub.execute_input":"2024-11-03T10:03:59.864616Z","iopub.status.idle":"2024-11-03T10:03:59.890548Z","shell.execute_reply.started":"2024-11-03T10:03:59.864554Z","shell.execute_reply":"2024-11-03T10:03:59.889237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create the submission DataFrame\nsubmission_df = pd.DataFrame({\n    'id': test_mean_values_df.index,\n    'sii': submission_predictions\n})\n\n# Save submission DataFrame to CSV\nsubmission_df.to_csv('submission.csv', index=False)\n\nprint(\"Submission file created:\")\nprint(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-03T10:04:13.614335Z","iopub.execute_input":"2024-11-03T10:04:13.614832Z","iopub.status.idle":"2024-11-03T10:04:13.633816Z","shell.execute_reply.started":"2024-11-03T10:04:13.614787Z","shell.execute_reply":"2024-11-03T10:04:13.632432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}