{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import accuracy_score, classification_report \n\n\n# Load the data\ntrain_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest_df = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\ndata_dict = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')\nsample_submission = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:23:03.078026Z","iopub.execute_input":"2024-12-18T22:23:03.078437Z","iopub.status.idle":"2024-12-18T22:23:03.142107Z","shell.execute_reply.started":"2024-12-18T22:23:03.078405Z","shell.execute_reply":"2024-12-18T22:23:03.141265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Displaying the first few rows of each dataset\ntrain_head = train_df.head()\ntest_head = test_df.head()\ndata_dict_head = data_dict.head()\nsample_submission_head = sample_submission.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:23:17.005391Z","iopub.execute_input":"2024-12-18T22:23:17.006282Z","iopub.status.idle":"2024-12-18T22:23:17.011865Z","shell.execute_reply.started":"2024-12-18T22:23:17.006244Z","shell.execute_reply":"2024-12-18T22:23:17.010806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Column details\ntrain_columns = train_df.columns\ntest_columns = test_df.columns\n\n# Define a function to summarize missing data\ndef missing_data_summary(df):\n    \"\"\"\n    Summarizes missing data in a DataFrame.\n\n    Parameters:\n    df (DataFrame): The DataFrame to analyze.\n\n    Returns:\n    DataFrame: A DataFrame summarizing missing data.\n    \"\"\"\n    return (\n        pd.DataFrame(df.isna().sum())\n        .reset_index()\n        .rename(columns={'index': 'Column', 0: 'mis_count'})\n        .query('mis_count > 0')\n        .assign(Missing_Percentage=lambda x: (x['mis_count'] / df.shape[0]) * 100)\n        .sort_values('mis_count', ascending=False)\n        .reset_index(drop=True)\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:28:08.741926Z","iopub.execute_input":"2024-12-18T22:28:08.742334Z","iopub.status.idle":"2024-12-18T22:28:08.748947Z","shell.execute_reply.started":"2024-12-18T22:28:08.742300Z","shell.execute_reply":"2024-12-18T22:28:08.747893Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Display dataset summaries\nprint(\"\\nDataset Shapes:\")\nprint(f\"Train Dataset Shape: {train_df.shape}\")\nprint(f\"Test Dataset Shape: {test_df.shape}\")\nprint(f\"Data Dictionary Shape: {data_dict.shape}\")\nprint(f\"Sample Submission Shape: {sample_submission.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:28:54.379393Z","iopub.execute_input":"2024-12-18T22:28:54.379802Z","iopub.status.idle":"2024-12-18T22:28:54.385890Z","shell.execute_reply.started":"2024-12-18T22:28:54.379765Z","shell.execute_reply":"2024-12-18T22:28:54.384822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check for missing data\nprint(\"\\nMissing Data Summary - Train Dataset:\")\nprint(missing_data_summary(train_df))\n\nprint(\"\\nMissing Data Summary - Test Dataset:\")\nprint(missing_data_summary(test_df))\n\n# Identify columns in 'train_df' that are missing in 'test_df'\nmissing_columns = [col for col in train_df.columns if col not in test_df.columns]\nprint(\"\\nColumns in Train Dataset Missing from Test Dataset:\")\nprint(missing_columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:31:24.886632Z","iopub.execute_input":"2024-12-18T22:31:24.887021Z","iopub.status.idle":"2024-12-18T22:31:24.932647Z","shell.execute_reply.started":"2024-12-18T22:31:24.886988Z","shell.execute_reply":"2024-12-18T22:31:24.931469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Identify columns in 'train_df' that are missing in 'test_df'\nmissing_columns = [col for col in train_df.columns if col not in test_df.columns]\n\n# Create a DataFrame 'missing_data' for all data in 'train_df' columns missing from 'test_df'\nmissing_data = train_df[missing_columns]\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:40:54.967496Z","iopub.execute_input":"2024-12-18T22:40:54.967857Z","iopub.status.idle":"2024-12-18T22:40:54.975982Z","shell.execute_reply.started":"2024-12-18T22:40:54.967825Z","shell.execute_reply":"2024-12-18T22:40:54.974810Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Summarize missing data in the 'missing_data' DataFrame\nprint(\"\\nMissing Data Summary for Columns in Train but Missing in Test:\")\nprint(missing_data_summary(missing_data))\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:41:29.473837Z","iopub.execute_input":"2024-12-18T22:41:29.474208Z","iopub.status.idle":"2024-12-18T22:41:29.488175Z","shell.execute_reply.started":"2024-12-18T22:41:29.474177Z","shell.execute_reply":"2024-12-18T22:41:29.487008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensure 'pciat_columns' contains numeric data\npciat_columns = [col for col in train_df.columns if 'PCIAT' in col]  # Replace with your column selection logic\n\n# Convert columns to numeric, replacing non-numeric values with NaN and filling with 0\ntrain_df[pciat_columns] = train_df[pciat_columns].apply(pd.to_numeric, errors='coerce').fillna(0)\n\n# Update the 'PCIAT-PCIAT_Total' column with the sum of numeric values\ntrain_df['PCIAT-PCIAT_Total'] = train_df[pciat_columns].sum(axis=1)\n\n# Confirm the updated values\nprint(\"\\nUpdated 'PCIAT-PCIAT_Total' Column:\")\nprint(train_df['PCIAT-PCIAT_Total'].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:41:38.461998Z","iopub.execute_input":"2024-12-18T22:41:38.462406Z","iopub.status.idle":"2024-12-18T22:41:38.485668Z","shell.execute_reply.started":"2024-12-18T22:41:38.462372Z","shell.execute_reply":"2024-12-18T22:41:38.484673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Confirm the updated values for 'PCIAT-PCIAT_Total'\nprint(\"\\nUpdated 'PCIAT-PCIAT_Total' Column:\")\nprint(train_df['PCIAT-PCIAT_Total'].head())\n\n# Update the 'sii' column based on rules defined in the data dictionary\ntrain_df['sii'] = 0  # Default to 0 (None)\ntrain_df.loc[(train_df['PCIAT-PCIAT_Total'] > 30) & (train_df['PCIAT-PCIAT_Total'] <= 49), 'sii'] = 1  # Mild\ntrain_df.loc[(train_df['PCIAT-PCIAT_Total'] > 49) & (train_df['PCIAT-PCIAT_Total'] <= 79), 'sii'] = 2  # Moderate\ntrain_df.loc[train_df['PCIAT-PCIAT_Total'] >= 80, 'sii'] = 3  # Severe","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:42:05.028959Z","iopub.execute_input":"2024-12-18T22:42:05.029366Z","iopub.status.idle":"2024-12-18T22:42:05.040970Z","shell.execute_reply.started":"2024-12-18T22:42:05.029332Z","shell.execute_reply":"2024-12-18T22:42:05.039936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Confirm the updated values for 'sii'\nprint(\"\\nUpdated 'sii' Value Counts:\")\nprint(train_df['sii'].value_counts())\n\n# Check missing values in train and test datasets\ntrain_missing = train_df.isnull().sum().sort_values(ascending=False)\ntest_missing = test_df.isnull().sum().sort_values(ascending=False)\n\n# Output the missing value summary for train and test datasets\ntrain_missing_summary = train_missing[train_missing > 0]\ntest_missing_summary = test_missing[test_missing > 0]\n\nprint(\"\\nMissing Value Summary - Train Dataset (Top 10):\")\nprint(train_missing_summary.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:42:21.831805Z","iopub.execute_input":"2024-12-18T22:42:21.832227Z","iopub.status.idle":"2024-12-18T22:42:21.851678Z","shell.execute_reply.started":"2024-12-18T22:42:21.832189Z","shell.execute_reply":"2024-12-18T22:42:21.850629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nMissing Value Summary - Test Dataset (Top 10):\")\nprint(test_missing_summary.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:42:33.239174Z","iopub.execute_input":"2024-12-18T22:42:33.239651Z","iopub.status.idle":"2024-12-18T22:42:33.245809Z","shell.execute_reply.started":"2024-12-18T22:42:33.239618Z","shell.execute_reply":"2024-12-18T22:42:33.244692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualizing the target variable distribution\nplt.figure(figsize=(8, 6))\nsns.countplot(x='sii', data=train_df)\nplt.title(\"Distribution of Target Variable (sii)\")\nplt.xlabel(\"sii\")\nplt.ylabel(\"Count\")\nplt.show()\n# Comment: The distribution helps us understand class imbalance in the target variable.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:42:47.455414Z","iopub.execute_input":"2024-12-18T22:42:47.455801Z","iopub.status.idle":"2024-12-18T22:42:47.689943Z","shell.execute_reply.started":"2024-12-18T22:42:47.455768Z","shell.execute_reply":"2024-12-18T22:42:47.688951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualizing Age Distribution\nplt.figure(figsize=(10, 6))\nsns.histplot(train_df['Basic_Demos-Age'], bins=20, kde=True)\nplt.title(\"Age Distribution of Participants\")\nplt.xlabel(\"Age\")\nplt.ylabel(\"Count\")\nplt.show()\n# Comment: Understanding the age range of participants and its possible impact on target.\n\n# Visualizing Correlation Between Numeric Features\nplt.figure(figsize=(12, 8))\ncorrelation = train_df.select_dtypes(include=[np.number]).corr()\nsns.heatmap(correlation, annot=False, cmap=\"coolwarm\")\nplt.title(\"Correlation Matrix for Numeric Features\")\nplt.show()\n# Comment: Identifying multicollinearity or relationships among numeric features.\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:43:06.304180Z","iopub.execute_input":"2024-12-18T22:43:06.304556Z","iopub.status.idle":"2024-12-18T22:43:07.546415Z","shell.execute_reply.started":"2024-12-18T22:43:06.304522Z","shell.execute_reply":"2024-12-18T22:43:07.545365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EDA: Exploratory Data Analysis\n# Overview of the data\nprint(\"Train dataset shape:\", train_df.shape)\nprint(\"Test dataset shape:\", test_df.shape)\nprint(\"Data Dictionary Overview:\", data_dict.head())\n\n# Data Engineering/Prep Steps\n# Separate numeric and categorical columns\nnumeric_cols = train_df.select_dtypes(include=['number']).columns\ncategorical_cols = train_df.select_dtypes(include=['object']).columns\n\n# Handle missing values\nimputer_num = SimpleImputer(strategy=\"median\")\nimputer_cat = SimpleImputer(strategy=\"most_frequent\")\n\nX_numeric = pd.DataFrame(imputer_num.fit_transform(train_df[numeric_cols]), columns=numeric_cols)\nX_categorical = pd.DataFrame(imputer_cat.fit_transform(train_df[categorical_cols]), columns=categorical_cols)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:43:33.785176Z","iopub.execute_input":"2024-12-18T22:43:33.786051Z","iopub.status.idle":"2024-12-18T22:43:33.857210Z","shell.execute_reply.started":"2024-12-18T22:43:33.786013Z","shell.execute_reply":"2024-12-18T22:43:33.856152Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Encode categorical columns\nlabel_encoders = {}\nfor col in X_categorical.columns:\n    le = LabelEncoder()\n    X_categorical[col] = le.fit_transform(X_categorical[col])\n    label_encoders[col] = le\n\n\n# Combine numeric and categorical features\nX_preprocessed = pd.concat([X_numeric, X_categorical], axis=1)\ny = train_df['sii'].fillna(train_df['sii'].median())  # Target variable\n\n# Test/Train Split\nX_train, X_val, y_train, y_val = train_test_split(X_preprocessed, y, test_size=0.2, random_state=42)\n\n# Model Training\n# Choosing RandomForestClassifier as the predictive algorithm for its robustness with mixed data\nrf_model = RandomForestClassifier(n_estimators=100, random_state=42)\nrf_model.fit(X_train, y_train)\n\n# Model Training\nrf_model = RandomForestClassifier(n_estimators=100, random_state=42)\nrf_model.fit(X_train, y_train)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:44:21.997569Z","iopub.execute_input":"2024-12-18T22:44:21.998321Z","iopub.status.idle":"2024-12-18T22:44:23.145831Z","shell.execute_reply.started":"2024-12-18T22:44:21.998281Z","shell.execute_reply":"2024-12-18T22:44:23.144786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Prediction and Accuracy Results\ny_pred = rf_model.predict(X_val)\naccuracy = accuracy_score(y_val, y_pred)  # Calculate accuracy\nclassification_rep = classification_report(y_val, y_pred)\nprint(\"Validation Accuracy:\", accuracy)\nprint(\"Classification Report:\\n\", classification_rep)\n\n# Prediction and Accuracy Results\ny_pred = rf_model.predict(X_val)\naccuracy = accuracy_score(y_val, y_pred)  # Calculate accuracy\nclassification_rep = classification_report(y_val, y_pred)\nprint(\"Validation Accuracy:\", accuracy)\nprint(\"Classification Report:\\n\", classification_rep)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:54:07.328893Z","iopub.execute_input":"2024-12-18T22:54:07.329825Z","iopub.status.idle":"2024-12-18T22:54:07.383256Z","shell.execute_reply.started":"2024-12-18T22:54:07.329785Z","shell.execute_reply":"2024-12-18T22:54:07.382141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare Test Data\n# Ensure all numeric and categorical columns are present\nfor col in numeric_cols:\n    if col not in test_df.columns:\n        test_df[col] = np.nan  # Add missing numeric columns\n\nfor col in categorical_cols:\n    if col not in test_df.columns:\n        test_df[col] = \"Unknown\"  # Add missing categorical columns\n\n# Transform numeric columns\ntest_numeric = pd.DataFrame(imputer_num.transform(test_df[numeric_cols]), columns=numeric_cols)\n\n# Transform categorical columns\ntest_categorical = pd.DataFrame(imputer_cat.transform(test_df[categorical_cols]), columns=categorical_cols)\n\n# Handle unseen categories in categorical data\nfor col in test_categorical.columns:\n    test_categorical[col] = test_categorical[col].apply(\n        lambda x: x if x in label_encoders[col].classes_ else label_encoders[col].classes_[0]\n    )\n    test_categorical[col] = label_encoders[col].transform(test_categorical[col])\n\n# Combine numeric and categorical features\nX_test = pd.concat([test_numeric, test_categorical], axis=1)\n\n# Ensure X_test columns match X_preprocessed\nfor col in X_preprocessed.columns:\n    if col not in X_test.columns:\n        X_test[col] = 0  # Add missing columns with default value\n\nX_test = X_test[X_preprocessed.columns]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T22:57:01.750881Z","iopub.execute_input":"2024-12-18T22:57:01.751306Z","iopub.status.idle":"2024-12-18T22:57:01.791056Z","shell.execute_reply.started":"2024-12-18T22:57:01.751270Z","shell.execute_reply":"2024-12-18T22:57:01.790005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Test Predictions\ntest_predictions = rf_model.predict(X_test)\n\n# Submission\nsubmission = pd.DataFrame({\n    'id': test_df['id'],\n    'sii': test_predictions,\n})\nsubmission['accuracy'] = accuracy  # Add accuracy to the DataFrame\nsubmission.to_csv(\"submission.csv\", index=False)\n\n# Remove the 'accuracy' column\nsubmission = submission.drop(columns=['accuracy'])\n\n# Save the updated DataFrame to CSV\nsubmission.to_csv(\"submission.csv\", index=False)\n\n# Display final results without index\nprint(\"Submission Preview:\\n\", submission.to_string(index=False))\nprint(\"Submission saved as submission.csv\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-18T23:18:25.923962Z","iopub.execute_input":"2024-12-18T23:18:25.924389Z","iopub.status.idle":"2024-12-18T23:18:25.946067Z","shell.execute_reply.started":"2024-12-18T23:18:25.924353Z","shell.execute_reply":"2024-12-18T23:18:25.945140Z"}},"outputs":[],"execution_count":null}]}