{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-07T04:48:52.165093Z","iopub.execute_input":"2024-10-07T04:48:52.165579Z","iopub.status.idle":"2024-10-07T04:48:55.206713Z","shell.execute_reply.started":"2024-10-07T04:48:52.165531Z","shell.execute_reply":"2024-10-07T04:48:55.204942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import MinMaxScaler\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\n# Import necessary libraries\nimport xgboost as xgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.ensemble import VotingRegressor\nimport matplotlib.pyplot as plt\n\nimport pandas as pd\nfrom sklearn.cluster import KMeans\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport glob\nimport pdb","metadata":{"execution":{"iopub.status.busy":"2024-10-07T04:48:55.208812Z","iopub.execute_input":"2024-10-07T04:48:55.209362Z","iopub.status.idle":"2024-10-07T04:48:56.943064Z","shell.execute_reply.started":"2024-10-07T04:48:55.209316Z","shell.execute_reply":"2024-10-07T04:48:56.94189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\ndata_dict = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-07T04:48:56.944479Z","iopub.execute_input":"2024-10-07T04:48:56.944979Z","iopub.status.idle":"2024-10-07T04:48:57.036664Z","shell.execute_reply.started":"2024-10-07T04:48:56.944937Z","shell.execute_reply":"2024-10-07T04:48:57.035637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef process_parquet(file_path):\n    # Read the Parquet file into a DataFrame\n    df = pd.read_parquet(file_path)\n\n    # Compute features\n    mean_x = df['X'].mean()\n    mean_y = df['Y'].mean()\n    mean_z = df['Z'].mean()\n    mean_x_abs = df['X'].abs().mean()\n    mean_y_abs = df['Y'].abs().mean()\n    mean_z_abs = df['Z'].abs().mean()\n\n\n    std_x = df['X'].std()\n    std_y = df['Y'].std()\n    std_z = df['Z'].std()\n\n    mean_enmo = df['enmo'].mean()\n    max_enmo = df['enmo'].max()\n    min_enmo = df['enmo'].min()\n    total_activity = df['enmo'].sum()\n\n    resting_periods = df[df['enmo'] == 0].shape[0]\n\n    # Convert 'time_of_day' to timedelta and extract hour\n    df['time_of_day'] = pd.to_timedelta(df['time_of_day'], unit='ns')\n    df['hour'] = df['time_of_day'].dt.components['hours']\n\n    # Compute average activity for different times of day\n    morning_activity = df[df['hour'] < 12]['enmo'].mean()\n    afternoon_activity = df[(df['hour'] >= 12) & (df['hour'] < 18)]['enmo'].mean()\n    evening_activity = df[df['hour'] >= 18]['enmo'].mean()\n\n    non_wear_duration = df[df['non-wear_flag'] == 1].shape[0] * 5  # In seconds\n\n    # Calculate magnitude\n    df['magnitude'] = (df['X']**2 + df['Y']**2 + df['Z']**2)**0.5\n    mean_magnitude = df['magnitude'].mean()\n\n    mean_anglez = df['anglez'].mean()\n    max_anglez = df['anglez'].max()\n\n    # Daily features\n    df['day'] = df['relative_date_PCIAT']\n \n    # Active ratio calculation\n    threshold = 0.1\n    active_periods = df[df['enmo'] > threshold].shape[0]\n    total_periods = df.shape[0]\n    active_ratio = active_periods / total_periods\n\n    # Return the aggregated features in a dictionary\n    return {\n        'mean_x': mean_x,\n        'mean_y': mean_y,\n        'mean_z': mean_z,\n        'mean_x_abs': mean_x_abs,\n        'mean_y_abs': mean_y_abs,\n        'mean_z_abs': mean_z_abs,\n        'std_x': std_x,\n        'std_y': std_y,\n        'std_z': std_z,\n        'mean_enmo': mean_enmo,\n        'max_enmo': max_enmo,\n        'min_enmo': min_enmo,\n        'total_activity': total_activity,\n        'resting_periods': resting_periods,\n        'morning_activity': morning_activity,\n        'afternoon_activity': afternoon_activity,\n        'evening_activity': evening_activity,\n        'non_wear_duration': non_wear_duration,\n        'mean_magnitude': mean_magnitude,\n        'mean_anglez': mean_anglez,\n        'max_anglez': max_anglez,\n        'active_ratio': active_ratio,\n    }\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T04:48:57.039564Z","iopub.execute_input":"2024-10-07T04:48:57.039967Z","iopub.status.idle":"2024-10-07T04:48:57.05752Z","shell.execute_reply.started":"2024-10-07T04:48:57.039924Z","shell.execute_reply":"2024-10-07T04:48:57.056255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nparquet_files = glob.glob('/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet/id=*/part-0.parquet')\nparquet_files_test = glob.glob('/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet/id=*/part-0.parquet')\n\n\n# Initialize an empty list to store features for each user\nfeatures_list = []\nfeatures_list_test = []\n\n# Loop through each train file and process\nfor file in tqdm(parquet_files, desc=\"Processing Parquet train files\"):\n    # Normalize the path and split by the OS separator\n    path_parts = os.path.normpath(file).split(os.sep)\n    user_id_full = path_parts[-2]  # This will give 'id=00115b9f'\n    \n    # Extract the part after 'id='\n    user_id = user_id_full.split('=')[-1]\n    \n    # Process parquet file to extract features\n    features = process_parquet(file)\n    features['id'] = user_id  # Add the user ID to the features dictionary\n    features_list.append(features) \n\n\n# Loop through each test file and process\nfor file in tqdm(parquet_files_test, desc=\"Processing Parquet test files\"):\n    # Normalize the path and split by the OS separator\n    path_parts = os.path.normpath(file).split(os.sep)\n    user_id_full = path_parts[-2]  # This will give 'id=00115b9f'\n    \n    # Extract the part after 'id='\n    user_id = user_id_full.split('=')[-1]\n    \n    # Process parquet file to extract features\n    features = process_parquet(file)\n    features['id'] = user_id  # Add the user ID to the features dictionary\n    features_list_test.append(features)\n\n\n# Convert the features list to DataFrames\nfeatures_df = pd.DataFrame(features_list)\nfeatures_df_test = pd.DataFrame(features_list_test)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T04:48:57.059221Z","iopub.execute_input":"2024-10-07T04:48:57.060155Z","iopub.status.idle":"2024-10-07T05:42:10.625845Z","shell.execute_reply.started":"2024-10-07T04:48:57.060051Z","shell.execute_reply":"2024-10-07T05:42:10.624444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = len(train) * 0.3\ndf_cleaned = train.dropna(thresh=threshold, axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:18.548059Z","iopub.execute_input":"2024-10-07T06:04:18.548558Z","iopub.status.idle":"2024-10-07T06:04:18.56691Z","shell.execute_reply.started":"2024-10-07T06:04:18.548512Z","shell.execute_reply":"2024-10-07T06:04:18.565184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge the features DataFrame with the train DataFrame\ntrain_merged = df_cleaned.merge(features_df, on='id', how='left')\n\ntest_merged = test.merge(features_df_test, on='id', how='left')\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:19.122067Z","iopub.execute_input":"2024-10-07T06:04:19.122555Z","iopub.status.idle":"2024-10-07T06:04:19.143246Z","shell.execute_reply.started":"2024-10-07T06:04:19.122507Z","shell.execute_reply":"2024-10-07T06:04:19.141602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove rows where 'sii' has null values\ndf_sii_nonnull = train_merged.dropna(subset=['sii'])\ndf_sii_nonnull_test = test_merged\n\ndf_sii_nonnull.info()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:19.912223Z","iopub.execute_input":"2024-10-07T06:04:19.912732Z","iopub.status.idle":"2024-10-07T06:04:19.942178Z","shell.execute_reply.started":"2024-10-07T06:04:19.912682Z","shell.execute_reply":"2024-10-07T06:04:19.940855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t1_train = df_sii_nonnull.drop(['FGC-FGC_SRL', 'Physical-BMI', 'BIA-BIA_ECW', 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST','Fitness_Endurance-Season'], axis=1)\nt1_test = df_sii_nonnull_test.drop(['FGC-FGC_SRL', 'Physical-BMI', 'BIA-BIA_ECW', 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST','Fitness_Endurance-Season'], axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:20.997063Z","iopub.execute_input":"2024-10-07T06:04:20.997601Z","iopub.status.idle":"2024-10-07T06:04:21.008393Z","shell.execute_reply.started":"2024-10-07T06:04:20.997555Z","shell.execute_reply":"2024-10-07T06:04:21.006917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = t1_train\ndf_test = t1_test\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:21.882169Z","iopub.execute_input":"2024-10-07T06:04:21.882624Z","iopub.status.idle":"2024-10-07T06:04:21.909474Z","shell.execute_reply.started":"2024-10-07T06:04:21.88258Z","shell.execute_reply":"2024-10-07T06:04:21.908166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"physical_activity_columns = [\n    'mean_x', 'mean_y', 'mean_z','mean_x_abs', 'mean_y_abs', 'mean_z_abs', 'mean_enmo',\n    'std_x', 'std_y', 'std_z', 'max_enmo', 'min_enmo',\n    'total_activity', 'resting_periods', 'morning_activity',\n    'afternoon_activity', 'evening_activity', 'non_wear_duration',\n    'mean_magnitude', 'mean_anglez', 'max_anglez', 'active_ratio'\n]\ngrouped_means = df.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex'])[physical_activity_columns].transform('mean')\ngrouped_means_test = df_test.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex'])[physical_activity_columns].transform('mean')\n\ndf[physical_activity_columns] = df[physical_activity_columns].fillna(grouped_means)\ndf_test[physical_activity_columns] = df_test[physical_activity_columns].fillna(grouped_means_test)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:22.627302Z","iopub.execute_input":"2024-10-07T06:04:22.627797Z","iopub.status.idle":"2024-10-07T06:04:22.666114Z","shell.execute_reply.started":"2024-10-07T06:04:22.627743Z","shell.execute_reply":"2024-10-07T06:04:22.664609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Age', 'Basic_Demos-Sex', 'CGAS-CGAS_Score', 'Physical-Height', 'Physical-Weight', 'Physical-Diastolic_BP', 'Physical-HeartRate', \n                'Physical-Systolic_BP', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_PU', 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone', \n                'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', \n                'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num', 'BIA-BIA_SMM', 'BIA-BIA_TBW', 'PAQ_C-PAQ_C_Total', 'PCIAT-PCIAT_01', 'PCIAT-PCIAT_02', 'PCIAT-PCIAT_03', \n                'PCIAT-PCIAT_04', 'PCIAT-PCIAT_05', 'PCIAT-PCIAT_06', 'PCIAT-PCIAT_07', 'PCIAT-PCIAT_08', 'PCIAT-PCIAT_09', 'PCIAT-PCIAT_10', 'PCIAT-PCIAT_11', 'PCIAT-PCIAT_12',\n                'PCIAT-PCIAT_13', 'PCIAT-PCIAT_14', 'PCIAT-PCIAT_15', 'PCIAT-PCIAT_16', 'PCIAT-PCIAT_17', 'PCIAT-PCIAT_18', 'PCIAT-PCIAT_19', 'PCIAT-PCIAT_20', \n                'PCIAT-PCIAT_Total', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T', 'PreInt_EduHx-computerinternet_hoursday', 'sii', 'mean_x', 'mean_y', 'mean_z', 'mean_x_abs', \n                'mean_y_abs', 'mean_z_abs', 'std_x', 'std_y', 'std_z', 'mean_enmo', 'max_enmo', 'min_enmo', 'total_activity', 'resting_periods', 'morning_activity', \n                'afternoon_activity', 'evening_activity', 'non_wear_duration', 'mean_magnitude', 'mean_anglez', 'max_anglez', 'active_ratio']\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'FGC-Season', 'BIA-Season', 'PAQ_C-Season', 'PCIAT-Season', 'SDS-Season', 'PreInt_EduHx-Season']\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    if col in df.columns and col in df_test.columns:\n        mapping = create_mapping(col, df)\n        mappingTe = create_mapping(col, df_test)\n\n        df[col] = df[col].replace(mapping).astype(int)\n        df_test[col] = df_test[col].replace(mappingTe).astype(int)\n    else:\n        print(f\"Column {col} not found in one of the datasets.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:23.574345Z","iopub.execute_input":"2024-10-07T06:04:23.574809Z","iopub.status.idle":"2024-10-07T06:04:23.6353Z","shell.execute_reply.started":"2024-10-07T06:04:23.574763Z","shell.execute_reply":"2024-10-07T06:04:23.63414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['non_wear_duration'] = df['non_wear_duration'] /(3600*24)\ndf_test['non_wear_duration'] = df_test['non_wear_duration'] /(3600*24)","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:24.527115Z","iopub.execute_input":"2024-10-07T06:04:24.527591Z","iopub.status.idle":"2024-10-07T06:04:24.535419Z","shell.execute_reply.started":"2024-10-07T06:04:24.527545Z","shell.execute_reply":"2024-10-07T06:04:24.533962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in featuresCols:\n    if col in df.columns and col in df_test.columns:  # Ensure the column exists in both datasets\n        if df[col].dtype == 'object':  # For categorical data\n            df[col].fillna(df[col].mode()[0], inplace=True)\n            df_test[col].fillna(df_test[col].mode()[0], inplace=True)\n        else:  # For numerical data\n            df[col].fillna(df[col].median(), inplace=True)\n            df_test[col].fillna(df_test[col].median(), inplace=True)\n    else:\n        print(f\"Column {col} not found in one of the datasets.\")\n","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-10-07T06:04:25.587117Z","iopub.execute_input":"2024-10-07T06:04:25.587577Z","iopub.status.idle":"2024-10-07T06:04:25.659549Z","shell.execute_reply.started":"2024-10-07T06:04:25.587534Z","shell.execute_reply":"2024-10-07T06:04:25.658363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = df_test.drop(columns=['Physical-Waist_Circumference','PAQ_A-Season' ])\ntest_col= df_test.columns\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:26.747138Z","iopub.execute_input":"2024-10-07T06:04:26.747577Z","iopub.status.idle":"2024-10-07T06:04:26.756117Z","shell.execute_reply.started":"2024-10-07T06:04:26.747537Z","shell.execute_reply":"2024-10-07T06:04:26.754828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:29.546943Z","iopub.execute_input":"2024-10-07T06:04:29.547447Z","iopub.status.idle":"2024-10-07T06:04:29.569981Z","shell.execute_reply.started":"2024-10-07T06:04:29.5474Z","shell.execute_reply":"2024-10-07T06:04:29.568121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_columns = df.columns.intersection(df_test.columns)\ncommon_columns","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:04:36.722306Z","iopub.execute_input":"2024-10-07T06:04:36.722828Z","iopub.status.idle":"2024-10-07T06:04:36.733165Z","shell.execute_reply.started":"2024-10-07T06:04:36.722781Z","shell.execute_reply":"2024-10-07T06:04:36.731708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nX_train = df[common_columns].drop(columns=['id'])  # Features\ny_train = df['sii']  # Target\n\nid_column = df_test['id']\n\nxgb_model = xgb.XGBClassifier(objective='multi:softmax', num_class=4, random_state=42)\n\nxgb_model.fit(X_train, y_train)\n\n\n# Split the dataset into training and testing sets (80% train, 20% test)\nX_test_final = df_test[common_columns].drop(columns=['id'])  # Exclude 'sii' if it's present in the test set\ny_test_pred = xgb_model.predict(X_test_final)\n\nsubmission_df = pd.DataFrame({\n    'id': id_column,\n    'sii': y_test_pred\n})\n\n# Save the DataFrame to a CSV file in the required format\nsubmission_df.to_csv('submission.csv', index=False)\n\n# Evaluate model performance\n# mse = mean_squared_error(y_test, y_pred)\n# r2 = r2_score(y_test, y_pred)\n\n# print(f\"Mean Squared Error (MSE): {mse}\")\n# print(f\"R-squared (R2): {r2}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:10:31.392449Z","iopub.execute_input":"2024-10-07T06:10:31.393526Z","iopub.status.idle":"2024-10-07T06:10:32.67402Z","shell.execute_reply.started":"2024-10-07T06:10:31.393468Z","shell.execute_reply":"2024-10-07T06:10:32.672752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"execution":{"iopub.status.busy":"2024-10-07T06:10:33.126952Z","iopub.execute_input":"2024-10-07T06:10:33.127433Z","iopub.status.idle":"2024-10-07T06:10:33.140837Z","shell.execute_reply.started":"2024-10-07T06:10:33.127387Z","shell.execute_reply":"2024-10-07T06:10:33.139498Z"},"trusted":true},"execution_count":null,"outputs":[]}]}