{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.feature_selection import SelectKBest, f_classif\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn.pipeline import Pipeline\n\n# Loading the data\ntrain_data = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ndata_dict = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')\n\n# 1. Feature Identification\nrelevant_features = [\n    'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI', 'Physical-Height', 'Physical-Weight',\n    'Physical-Waist_Circumference', 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n    'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n    'FGC-FGC_CU', 'FGC-FGC_GSND', 'FGC-FGC_GSD', 'FGC-FGC_PU', 'FGC-FGC_SRL', 'FGC-FGC_SRR', 'FGC-FGC_TL',\n    'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE',\n    'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n    'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW',\n    'PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T'\n]\n\ntrain_data = train_data.dropna(subset=['sii'])\n\nX = train_data[relevant_features]\ny = train_data['sii']\n\n# 2. Handling Missing Data\n\n# Imputing missing values\nimputer = SimpleImputer(strategy='median')\nX_imputed = pd.DataFrame(imputer.fit_transform(X), columns=X.columns)\n\n# 3. Encoding Categorical Variables\n\nle = LabelEncoder()\nX_imputed['Basic_Demos-Sex'] = le.fit_transform(X_imputed['Basic_Demos-Sex'])\n\n# 4. Visualization\n# Correlation Heatmap\nplt.figure(figsize=(12, 8))\nsns.heatmap(X_imputed.corr(), cmap='coolwarm', annot=False)\nplt.title('Correlation Heatmap of Features')\nplt.tight_layout()\nplt.show()\n\n# Distribution of target variable sii\nplt.figure(figsize=(10, 6))\nsns.countplot(x='sii', data=train_data)\nplt.title('Distribution of Target Variable (sii)')\nplt.xlabel('Severity Impairment Index (sii)')\nplt.ylabel('Count')\nplt.show()\n\n# 5. Classification\n# Splitting the data\nX_train, X_test, y_train, y_test = train_test_split(X_imputed, y, test_size=0.2, random_state=42)\n\n# 6. Correlation Analysis\ncorrelation_matrix = X_imputed.corr()\nhigh_corr_features = np.where(np.abs(correlation_matrix) > 0.8)\nhigh_corr_features = [(correlation_matrix.index[x], correlation_matrix.columns[y]) \n                      for x, y in zip(*high_corr_features) if x != y and x < y]\nprint(\"Highly correlated features:\")\nprint(high_corr_features)\n\n# 7. Choosing a model\n# We'll use Random Forest as our model\nmodel = RandomForestClassifier(n_estimators=100, random_state=42)\n\n# 8. Training the Model\nmodel.fit(X_train, y_train)\n\n# 9. Feature Selection\nselector = SelectKBest(f_classif, k=20)\nX_new = selector.fit_transform(X_train, y_train)\nselected_features = X_train.columns[selector.get_support()].tolist()\nprint(\"Selected features:\", selected_features)\n\n# 10. Cross-Validation\ncv_scores = cross_val_score(model, X_train, y_train, cv=5)\nprint(\"Cross-validation scores:\", cv_scores)\nprint(\"Mean CV score:\", cv_scores.mean())\n\n# Creating a pipeline with feature selection and model\npipeline = Pipeline([\n    ('selector', SelectKBest(f_classif, k=20)),\n    ('model', RandomForestClassifier(n_estimators=100, random_state=42))\n])\n\n# Fitting the pipeline\npipeline.fit(X_train, y_train)\n\n# Making predictions\ny_pred = pipeline.predict(X_test)\n\nprint(classification_report(y_test, y_pred))\n\nprint(confusion_matrix(y_test, y_pred))\n\n# Function to predict sii for test data\ndef predict_sii(test_data):\n    # Preparing test data\n    X_test = test_data[relevant_features]\n    X_test_imputed = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns)\n    X_test_imputed['Basic_Demos-Sex'] = le.transform(X_test_imputed['Basic_Demos-Sex'])\n    \n    # Making predictions\n    sii_pred = pipeline.predict(X_test_imputed)\n    \n    return sii_pred\n\n# Loading test data and making predictions\ntest_data = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\ntest_predictions = predict_sii(test_data)\n\n# Creating submission file\nsubmission = pd.DataFrame({\n    'id': test_data['id'],\n    'sii': test_predictions\n})\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission file created successfully!\")\n\n# 11. Additional Visualizations\n\n# Feature Importance\nfeature_importance = pipeline.named_steps['model'].feature_importances_\nselected_features = pipeline.named_steps['selector'].get_support()\nfeature_names = X.columns[selected_features]\n\nplt.figure(figsize=(12, 8))\nsns.barplot(x=feature_importance, y=feature_names)\nplt.title('Feature Importance')\nplt.xlabel('Importance')\nplt.ylabel('Features')\nplt.tight_layout()\nplt.show()\n\n# Confusion Matrix Heatmap\nplt.figure(figsize=(10, 8))\nsns.heatmap(confusion_matrix(y_test, y_pred), annot=True, fmt='d', cmap='Blues')\nplt.title('Confusion Matrix')\nplt.xlabel('Predicted')\nplt.ylabel('Actual')\nplt.tight_layout()\nplt.show()\n\n# Distribution of Predictions\nplt.figure(figsize=(10, 6))\nsns.countplot(x=y_pred)\nplt.title('Distribution of Predictions')\nplt.xlabel('Severity Impairment Index (sii)')\nplt.ylabel('Count')\nplt.tight_layout()\nplt.show()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-24T21:02:35.548721Z","iopub.execute_input":"2024-11-24T21:02:35.549106Z","iopub.status.idle":"2024-11-24T21:02:43.738861Z","shell.execute_reply.started":"2024-11-24T21:02:35.549071Z","shell.execute_reply":"2024-11-24T21:02:43.737567Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nimport os\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.feature_selection import SelectKBest, f_classif\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn.pipeline import Pipeline\n\ndef process_file(filename, dirname):\n    \"\"\"Process a single parquet file and extract statistics.\"\"\"\n    try:\n        data = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n        data.drop('step', axis=1, inplace=True)\n        return data.describe().values.reshape(-1), filename.split('=')[1]\n    except Exception as e:\n        print(f\"Error processing {filename}: {str(e)}\")\n        return None\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    \"\"\"Load and process all parquet files in parallel.\"\"\"\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), \n                          total=len(ids), desc=\"Processing parquet files\"))\n    \n    # Filter out None results from failed processing\n    results = [r for r in results if r is not None]\n    stats, indexes = zip(*results)\n    \n    data = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    data['id'] = indexes\n    return data\n\n# Load time series data\nprint(\"Loading train time series data...\")\ntrain_ts = load_time_series('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet')\nprint(\"Loading test time series data...\")\ntest_ts = load_time_series('/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet')\n\n# Get time series feature columns\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove('id')\n\n# Load main training data\ntrain_data = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest_data = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n\n# Merge time series features with main data\ntrain_data = train_data.merge(train_ts, on='id', how='left')\ntest_data = test_data.merge(test_ts, on='id', how='left')\n\n# Define relevant features (including time series features)\nrelevant_features = [\n    'Basic_Demos-Age', 'Basic_Demos-Sex', 'Physical-BMI', 'Physical-Height', 'Physical-Weight',\n    'Physical-Waist_Circumference', 'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n    'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n    'FGC-FGC_CU', 'FGC-FGC_GSND', 'FGC-FGC_GSD', 'FGC-FGC_PU', 'FGC-FGC_SRL', 'FGC-FGC_SRR', 'FGC-FGC_TL',\n    'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE',\n    'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n    'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW',\n    'PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T'\n] + time_series_cols\n\n# Clean data\ntrain_data = train_data.dropna(subset=['sii'])\n\n# Prepare features and target\nX = train_data[relevant_features]\ny = train_data['sii']\n\n# Handle missing values\nimputer = SimpleImputer(strategy='median')\nX_imputed = pd.DataFrame(imputer.fit_transform(X), columns=X.columns)\n\n# Encode categorical variables\nle = LabelEncoder()\nX_imputed['Basic_Demos-Sex'] = le.fit_transform(X_imputed['Basic_Demos-Sex'])\n\n# Split data\nX_train, X_test, y_train, y_test = train_test_split(X_imputed, y, test_size=0.2, random_state=42)\n\n# Create and train pipeline\npipeline = Pipeline([\n    ('selector', SelectKBest(f_classif, k=20)),\n    ('model', RandomForestClassifier(n_estimators=100, random_state=42))\n])\n\npipeline.fit(X_train, y_train)\n\n# Make predictions\ny_pred = pipeline.predict(X_test)\n\n# Print model performance\nprint(\"\\nModel Performance:\")\nprint(classification_report(y_test, y_pred))\nprint(\"\\nConfusion Matrix:\")\nprint(confusion_matrix(y_test, y_pred))\n\n# Prepare test predictions\nX_test_full = test_data[relevant_features]\nX_test_imputed = pd.DataFrame(imputer.transform(X_test_full), columns=X_test_full.columns)\nX_test_imputed['Basic_Demos-Sex'] = le.transform(X_test_imputed['Basic_Demos-Sex'])\ntest_predictions = pipeline.predict(X_test_imputed)\n\n# Create submission file\nsubmission = pd.DataFrame({\n    'id': test_data['id'],\n    'sii': test_predictions\n})\nsubmission.to_csv('submission.csv', index=False)\nprint(\"\\nSubmission file created successfully!\")\n\n# Visualize feature importance\nfeature_importance = pipeline.named_steps['model'].feature_importances_\nselected_features = pipeline.named_steps['selector'].get_support()\nfeature_names = X.columns[selected_features]\n\nplt.figure(figsize=(12, 8))\nsns.barplot(x=feature_importance, y=feature_names)\nplt.title('Feature Importance (Including Time Series Features)')\nplt.xlabel('Importance')\nplt.ylabel('Features')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-25T00:10:14.121010Z","iopub.execute_input":"2024-11-25T00:10:14.121568Z","iopub.status.idle":"2024-11-25T00:11:49.370675Z","shell.execute_reply.started":"2024-11-25T00:10:14.121502Z","shell.execute_reply":"2024-11-25T00:11:49.369523Z"},"trusted":true},"outputs":[],"execution_count":null}]}