{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import libraries\nimport numpy as np\nimport pandas as pd\nimport os\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\n# Set a random seed for reproducibility\nSEED = 42\nn_folds = 5\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-02T08:59:41.988082Z","iopub.execute_input":"2024-11-02T08:59:41.98851Z","iopub.status.idle":"2024-11-02T08:59:41.997491Z","shell.execute_reply.started":"2024-11-02T08:59:41.988475Z","shell.execute_reply":"2024-11-02T08:59:41.996243Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load and Process Time Series Data","metadata":{}},{"cell_type":"code","source":"# Function to process individual time series file\ndef extract_series_statistics(filename, folder_path):\n    data = pd.read_parquet(os.path.join(folder_path, filename, 'part-0.parquet'))\n    data.drop('step', axis=1, inplace=True)\n    # Return statistical summaries as features\n    return data.describe().values.flatten(), filename.split('=')[1]\n\n# Load and process time series data for all participants\ndef load_time_series_data(folder_path) -> pd.DataFrame:\n    participant_ids = os.listdir(folder_path)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: extract_series_statistics(fname, folder_path), \n                                         participant_ids), total=len(participant_ids)))\n    \n    stats, ids = zip(*results)\n    df = pd.DataFrame(stats, columns=[f\"feature_{i}\" for i in range(len(stats[0]))])\n    df['id'] = ids\n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T08:59:41.999445Z","iopub.execute_input":"2024-11-02T08:59:41.999836Z","iopub.status.idle":"2024-11-02T08:59:42.010396Z","shell.execute_reply.started":"2024-11-02T08:59:41.999798Z","shell.execute_reply":"2024-11-02T08:59:42.009117Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Build and Train Autoencoders for Feature Extraction","metadata":{}},{"cell_type":"code","source":"# Keras-based autoencoder\ndef keras_autoencoder(input_dim, encoding_dim):\n    input_layer = Input(shape=(input_dim,))\n    encoded = Dense(encoding_dim, activation='relu')(input_layer)\n    decoded = Dense(input_dim, activation='sigmoid')(encoded)\n    autoencoder = Model(inputs=input_layer, outputs=decoded)\n    encoder = Model(inputs=input_layer, outputs=encoded)\n    autoencoder.compile(optimizer=Adam(), loss='mse')\n    return autoencoder, encoder\n\n# TensorFlow-based autoencoder training\ndef encode_with_keras_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    scaled_data = scaler.fit_transform(df)\n    \n    autoencoder, encoder = keras_autoencoder(scaled_data.shape[1], encoding_dim)\n    autoencoder.fit(scaled_data, scaled_data, epochs=epochs, batch_size=batch_size, shuffle=True, verbose=1)\n    \n    encoded_data = encoder.predict(scaled_data)\n    return pd.DataFrame(encoded_data, columns=[f'encoded_{i+1}' for i in range(encoded_data.shape[1])])\n\n# PyTorch-based autoencoder\nclass TorchAutoEncoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(TorchAutoEncoder, self).__init__()\n        self.encoder = nn.Sequential(nn.Linear(input_dim, encoding_dim), nn.ReLU())\n        self.decoder = nn.Sequential(nn.Linear(encoding_dim, input_dim), nn.Sigmoid())\n    \n    def forward(self, x):\n        encoded = self.encoder(x)\n        return self.decoder(encoded)\n\n# Training the PyTorch autoencoder\ndef encode_with_torch_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    scaler = StandardScaler()\n    scaled_data = scaler.fit_transform(df)\n    data_tensor = torch.FloatTensor(scaled_data)\n    \n    model = TorchAutoEncoder(data_tensor.shape[1], encoding_dim)\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(model.parameters())\n    \n    for epoch in range(epochs):\n        for i in range(0, len(data_tensor), batch_size):\n            batch = data_tensor[i : i + batch_size]\n            optimizer.zero_grad()\n            loss = criterion(model(batch), batch)\n            loss.backward()\n            optimizer.step()\n        \n        if (epoch + 1) % 10 == 0:\n            print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}')\n    \n    with torch.no_grad():\n        encoded_data = model.encoder(data_tensor).numpy()\n        \n    return pd.DataFrame(encoded_data, columns=[f'encoded_{i+1}' for i in range(encoded_data.shape[1])])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T08:59:42.012262Z","iopub.execute_input":"2024-11-02T08:59:42.012653Z","iopub.status.idle":"2024-11-02T08:59:42.028077Z","shell.execute_reply.started":"2024-11-02T08:59:42.012616Z","shell.execute_reply":"2024-11-02T08:59:42.026996Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# Load data\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample_submission = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n# Process time series data\ntrain_ts = load_time_series_data(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series_data(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n# Encode time series features\nencoded_train_ts = encode_with_torch_autoencoder(train_ts.drop('id', axis=1), encoding_dim=96, epochs=100, batch_size=32)\n\n# Merge encoded features back into main train and test sets\nencoded_train_ts['id'] = train_ts['id']\ntrain = pd.merge(train, encoded_train_ts, on='id')\ntest = pd.merge(test, encoded_train_ts, on='id')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T08:59:42.030721Z","iopub.execute_input":"2024-11-02T08:59:42.031272Z","iopub.status.idle":"2024-11-02T09:01:21.074774Z","shell.execute_reply.started":"2024-11-02T08:59:42.03122Z","shell.execute_reply":"2024-11-02T09:01:21.073522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop the 'id' column after merging as it is no longer needed\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\n# Define the list of features (featuresCols) to use for training\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\n# Append the encoded time series columns\ntime_series_cols = [col for col in encoded_train_ts.columns if col != 'id']  # Ensure only autoencoder features are included\nfeaturesCols += time_series_cols\n\n# Select only the relevant columns in train\ntrain = train[featuresCols]\n# Drop rows with missing target values in train\ntrain = train.dropna(subset=['sii'])\n\n# Define categorical columns for processing\ncategorical_columns = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n                       'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n                       'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T09:01:21.077365Z","iopub.execute_input":"2024-11-02T09:01:21.077922Z","iopub.status.idle":"2024-11-02T09:01:21.094356Z","shell.execute_reply.started":"2024-11-02T09:01:21.077869Z","shell.execute_reply":"2024-11-02T09:01:21.09308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define a function to encode categorical features to integer codes\ndef encode_categorical_features(df, categorical_columns):\n    for col in categorical_columns:\n        df[col] = df[col].cat.codes  # Encode category as integer codes\n    return df\n\n# Define and preprocess features and target variable\ntarget_column = 'sii'\nfeature_columns = [col for col in train.columns if col != target_column]\n\ntrain = train.dropna(subset=[target_column])\n\ndef preprocess_categorical_features(df):\n    for col in categorical_columns:\n        df[col].fillna('Missing', inplace=True)\n        df[col] = df[col].astype('category')\n    return df\n\ntrain = preprocess_categorical_features(train)\ntest = preprocess_categorical_features(test)\n\n# Quadratic weighted kappa metric for evaluation\ndef calculate_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n# Convert categorical features to integer codes\ntrain = encode_categorical_features(train, categorical_columns)\ntest = encode_categorical_features(test, categorical_columns)\n\n# Train ensemble model with LightGBM, XGBoost, and CatBoost\n# Train ensemble model with LightGBM, XGBoost, and CatBoost\ndef train_ensemble_model(train_data, target, test_data):\n    X = train_data.drop(columns=[target])\n    y = train_data[target]\n    \n    skf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=SEED)\n    test_predictions = np.zeros((test_data.shape[0], n_folds))\n\n    # Initialize models with updated parameters for LightGBM\n    lightgbm = LGBMRegressor(\n        learning_rate=0.046,\n        max_depth=8,  # Reduced depth\n        num_leaves=64,  # Reduced number of leaves\n        min_data_in_leaf=20,  # Increase minimum data per leaf\n        min_gain_to_split=0.1,  # Set minimum gain for split\n        feature_fraction=0.8,  # Use subset of features for each iteration\n        bagging_fraction=0.8,  # Randomly select part of data without replacement\n        bagging_freq=1,  # Frequency of bagging\n        random_state=SEED,\n    )\n    xgboost = XGBRegressor(learning_rate=0.05, max_depth=6, enable_categorical=True)\n    catboost = CatBoostRegressor(learning_rate=0.05, depth=6, verbose=0)\n    \n    ensemble_model = VotingRegressor(estimators=[('lgbm', lightgbm), ('xgb', xgboost), ('cat', catboost)])\n\n    for fold, (train_idx, val_idx) in enumerate(skf.split(X, y)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n        ensemble_model.fit(X_train, y_train)\n        val_predictions = ensemble_model.predict(X_val).round(0).astype(int)\n        test_predictions[:, fold] = ensemble_model.predict(test_data)\n        \n        print(f\"Fold {fold + 1} Kappa Score:\", calculate_kappa(y_val, val_predictions))\n\n    # Average test predictions across folds\n    final_test_predictions = test_predictions.mean(axis=1).round(0).astype(int)\n    \n    submission = pd.DataFrame({'id': sample_submission['id'], 'sii': final_test_predictions})\n    return submission\n\n\nsubmission = train_ensemble_model(train, target_column, test)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T09:01:21.096413Z","iopub.execute_input":"2024-11-02T09:01:21.096837Z","iopub.status.idle":"2024-11-02T09:02:51.911126Z","shell.execute_reply.started":"2024-11-02T09:01:21.096798Z","shell.execute_reply":"2024-11-02T09:02:51.909193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Quadratic weighted kappa metric for evaluation\n# def calculate_kappa(y_true, y_pred):\n#     return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n# # Convert categorical features to integer codes\n# # Remove the 'id' column from train and test data\n# train = encode_categorical_features(train, categorical_columns)\n# test = encode_categorical_features(test, categorical_columns)\n\n# # Train ensemble model with LightGBM, XGBoost, and CatBoost","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-02T09:02:51.91234Z","iopub.status.idle":"2024-11-02T09:02:51.912755Z","shell.execute_reply.started":"2024-11-02T09:02:51.912553Z","shell.execute_reply":"2024-11-02T09:02:51.912572Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}}]}