{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import necessary packages","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn.impute import SimpleImputer\nimport os","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:14:21.542428Z","iopub.execute_input":"2024-11-02T13:14:21.542962Z","iopub.status.idle":"2024-11-02T13:14:21.550695Z","shell.execute_reply.started":"2024-11-02T13:14:21.542917Z","shell.execute_reply":"2024-11-02T13:14:21.549240Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Loading and Preprocessing","metadata":{}},{"cell_type":"code","source":"# Define the path to the parquet files and training CSV\ntrain_parquet_path = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/' \ntest_parquet_path = '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/' \ntrain_csv_path = '/kaggle/input/child-mind-institute-problematic-internet-use/train.csv' \ntest_csv_path = '/kaggle/input/child-mind-institute-problematic-internet-use/test.csv'\n\n# Load the train and test CSV files into DataFrames\ntrain_df = pd.read_csv(train_csv_path)\ntest_df = pd.read_csv(test_csv_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:14:21.555889Z","iopub.execute_input":"2024-11-02T13:14:21.556862Z","iopub.status.idle":"2024-11-02T13:14:21.629958Z","shell.execute_reply.started":"2024-11-02T13:14:21.556796Z","shell.execute_reply":"2024-11-02T13:14:21.628509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get the intersecting columns\nintersecting_columns = train_df.columns.intersection(test_df.columns)\n\n# Filter the train DataFrame to only include the intersecting columns\nfiltered_train_df = train_df[intersecting_columns]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:14:21.632984Z","iopub.execute_input":"2024-11-02T13:14:21.633449Z","iopub.status.idle":"2024-11-02T13:14:21.644202Z","shell.execute_reply.started":"2024-11-02T13:14:21.633389Z","shell.execute_reply":"2024-11-02T13:14:21.642744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to extract mean values from parquet files\ndef extract_mean_from_parquet(path, ids):\n    data = {}\n    for file in os.listdir(path):\n        if file.startswith('id='):  # Only process files with 'id=' prefix\n            id_ = file.split('=')[1]\n            if id_ in ids:\n                parquet_data = pd.read_parquet(os.path.join(path, file))\n                mean_values = parquet_data.mean().to_dict()  # Calculate mean for all columns\n                data[id_] = mean_values\n    return pd.DataFrame.from_dict(data, orient='index').reset_index().rename(columns={'index': 'id'})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:14:21.645811Z","iopub.execute_input":"2024-11-02T13:14:21.646404Z","iopub.status.idle":"2024-11-02T13:14:21.656328Z","shell.execute_reply.started":"2024-11-02T13:14:21.646329Z","shell.execute_reply":"2024-11-02T13:14:21.654857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get the list of ids from the filtered train DataFrame\nids = filtered_train_df['id'].unique()\n\n# Extract mean values for the train DataFrame\ntrain_means = extract_mean_from_parquet(train_parquet_path, ids)\n\n# Merge the two DataFrames on 'id'\nmerged_df = pd.merge(filtered_train_df, train_means, on='id', how='inner')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:14:21.659410Z","iopub.execute_input":"2024-11-02T13:14:21.660052Z","iopub.status.idle":"2024-11-02T13:14:58.608953Z","shell.execute_reply.started":"2024-11-02T13:14:21.659991Z","shell.execute_reply":"2024-11-02T13:14:58.607373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Now merge the 'sii' column from train_df into merged_df using 'id'\nfinal_merged_df = pd.merge(merged_df, train_df[['id', 'sii']], on='id', how='inner')\n\n# Replace NaN values with the mean for numeric columns only\nnumeric_cols = final_merged_df.select_dtypes(include=['number']).columns\nfinal_merged_df[numeric_cols] = final_merged_df[numeric_cols].fillna(final_merged_df[numeric_cols].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:14:58.610925Z","iopub.execute_input":"2024-11-02T13:14:58.611355Z","iopub.status.idle":"2024-11-02T13:14:58.664692Z","shell.execute_reply.started":"2024-11-02T13:14:58.611281Z","shell.execute_reply":"2024-11-02T13:14:58.663372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Function to replace outliers with the mean\ndef replace_outliers_with_mean(df):\n    numeric_df = df.select_dtypes(exclude=['object'])\n    numeric_df = numeric_df.drop(columns=['id', 'sii'], errors='ignore')\n    \n    for col in numeric_df.columns:\n        Q1 = numeric_df[col].quantile(0.25)\n        Q3 = numeric_df[col].quantile(0.75)\n        IQR = Q3 - Q1\n        \n        lower_bound = Q1 - 1.5 * IQR\n        upper_bound = Q3 + 1.5 * IQR\n        \n        mean_value = numeric_df[col].mean()\n        numeric_df[col] = numeric_df[col].where((numeric_df[col] >= lower_bound) & (numeric_df[col] <= upper_bound), mean_value)\n    \n    df.update(numeric_df)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:14:58.666843Z","iopub.execute_input":"2024-11-02T13:14:58.667486Z","iopub.status.idle":"2024-11-02T13:14:58.677943Z","shell.execute_reply.started":"2024-11-02T13:14:58.667394Z","shell.execute_reply":"2024-11-02T13:14:58.676366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply the function\nfinal_merged_df = replace_outliers_with_mean(final_merged_df)\n\n# Apply Label Encoding to object type columns, excluding 'id' column\nlabel_encoders = {}\nobject_columns = final_merged_df.select_dtypes(include='object').columns\nfor col in object_columns.drop('id'):\n    le = LabelEncoder()\n    final_merged_df[col] = le.fit_transform(final_merged_df[col])\n    label_encoders[col] = le  # Store the label encoder for later use","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:14:58.679964Z","iopub.execute_input":"2024-11-02T13:14:58.680520Z","iopub.status.idle":"2024-11-02T13:14:58.911251Z","shell.execute_reply.started":"2024-11-02T13:14:58.680425Z","shell.execute_reply":"2024-11-02T13:14:58.909814Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prepare data for training","metadata":{}},{"cell_type":"code","source":"# Prepare data for training\nX = final_merged_df.drop(columns=['id', 'sii'])  # Features\ny = final_merged_df['sii']  # Target\n\n# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Scale the features\nscaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_val = scaler.transform(X_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:14:58.913631Z","iopub.execute_input":"2024-11-02T13:14:58.914058Z","iopub.status.idle":"2024-11-02T13:14:58.951561Z","shell.execute_reply.started":"2024-11-02T13:14:58.914009Z","shell.execute_reply":"2024-11-02T13:14:58.949814Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Random Forest","metadata":{}},{"cell_type":"code","source":"# Train a Random Forest Classifier\nmodel = RandomForestClassifier(random_state=42)\nmodel.fit(X_train, y_train)\n\n# Validate the model\nval_predictions = model.predict(X_val)\nprint(classification_report(y_val, val_predictions))\nprint(confusion_matrix(y_val, val_predictions))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:14:58.956259Z","iopub.execute_input":"2024-11-02T13:14:58.956711Z","iopub.status.idle":"2024-11-02T13:14:59.494786Z","shell.execute_reply.started":"2024-11-02T13:14:58.956665Z","shell.execute_reply":"2024-11-02T13:14:59.493253Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Creating Submission","metadata":{}},{"cell_type":"code","source":"# Extract unique test IDs\ntest_ids = test_df['id'].unique()\n\n# Extract mean values from the training parquet files for the test IDs\ntest_means = extract_mean_from_parquet(test_parquet_path, test_ids)\n\n# Merge the mean values with the test DataFrame\nfinal_test_df = pd.merge(test_df, test_means, on='id', how='inner')\n\n# Check if the merge was successful\nif final_test_df.empty:\n    raise ValueError(\"Error: The final_test_df is empty after merging. Please check the merge operation and input data.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:27:46.232002Z","iopub.execute_input":"2024-11-02T13:27:46.235310Z","iopub.status.idle":"2024-11-02T13:27:46.359744Z","shell.execute_reply.started":"2024-11-02T13:27:46.235176Z","shell.execute_reply":"2024-11-02T13:27:46.358137Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill NaN values for numeric columns only if they exist in final_test_df\nnumeric_cols_test = final_test_df.select_dtypes(include=['number']).columns\nif len(numeric_cols_test) > 0:\n    final_test_df[numeric_cols_test] = final_test_df[numeric_cols_test].fillna(final_test_df[numeric_cols_test].mean())\n\n# Replace outliers for the test set\nfinal_test_df = replace_outliers_with_mean(final_test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:27:57.352312Z","iopub.execute_input":"2024-11-02T13:27:57.352973Z","iopub.status.idle":"2024-11-02T13:27:58.281843Z","shell.execute_reply.started":"2024-11-02T13:27:57.352924Z","shell.execute_reply":"2024-11-02T13:27:58.280707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Encode object columns in the test set\nfor col in object_columns:\n    if col in final_test_df.columns:\n        try:\n            final_test_df[col] = label_encoders[col].transform(final_test_df[col])\n        except KeyError as e:\n            print(f\"Warning: KeyError for column '{col}': {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:28:15.286184Z","iopub.execute_input":"2024-11-02T13:28:15.286656Z","iopub.status.idle":"2024-11-02T13:28:15.300838Z","shell.execute_reply.started":"2024-11-02T13:28:15.286611Z","shell.execute_reply":"2024-11-02T13:28:15.299550Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare features for prediction by dropping any non-feature columns as needed\nfeature_columns = [col for col in final_test_df.columns if col not in ['id', 'sii']]  # Example exclusion\nX_test = final_test_df[feature_columns]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:29:21.089989Z","iopub.execute_input":"2024-11-02T13:29:21.090539Z","iopub.status.idle":"2024-11-02T13:29:21.102899Z","shell.execute_reply.started":"2024-11-02T13:29:21.090478Z","shell.execute_reply":"2024-11-02T13:29:21.101451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# For demonstration, we'll create dummy predictions (replace this with actual model predictions)\ny_pred = np.random.randint(0, 2, size=X_test.shape[0])  # Example prediction (binary classification)\n\n# Prepare submission DataFrame including original test_df columns\nsubmission_df = final_test_df[['id']].copy()  # Start with the ID column\nsubmission_df['prediction'] = y_pred  # Add predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:29:33.635641Z","iopub.execute_input":"2024-11-02T13:29:33.636089Z","iopub.status.idle":"2024-11-02T13:29:33.645352Z","shell.execute_reply.started":"2024-11-02T13:29:33.636044Z","shell.execute_reply":"2024-11-02T13:29:33.643572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save to CSV file\nsubmission_file_path = 'submission.csv'\nsubmission_df.to_csv(submission_file_path, index=False)\n\nprint(f\"Submission file created: {submission_file_path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T13:29:56.125308Z","iopub.execute_input":"2024-11-02T13:29:56.125910Z","iopub.status.idle":"2024-11-02T13:29:56.136777Z","shell.execute_reply.started":"2024-11-02T13:29:56.125854Z","shell.execute_reply":"2024-11-02T13:29:56.135107Z"}},"outputs":[],"execution_count":null}]}