{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":1259.762308,"end_time":"2024-10-16T01:29:25.368389","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-10-16T01:08:25.606081","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Trying out a new way of loading time series data**\n\n\n\n","metadata":{"papermill":{"duration":0.023757,"end_time":"2024-10-16T01:08:28.971241","exception":false,"start_time":"2024-10-16T01:08:28.947484","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# The score might fluctuate from 0.483 to 0.487 due to some randomness ","metadata":{"papermill":{"duration":0.025411,"end_time":"2024-10-16T01:08:29.020263","exception":false,"start_time":"2024-10-16T01:08:28.994852","status":"completed"},"tags":[]}},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":5.007811,"end_time":"2024-10-16T01:08:39.769786","exception":false,"start_time":"2024-10-16T01:08:34.761975","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy.stats import kurtosis\nimport numpy as np\nimport polars as pl\nimport pandas as pd\nimport pandas as pd\nimport numpy as np\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\n\nfrom sklearn.base import clone\nfrom copy import deepcopy\nimport optuna\nfrom scipy.optimize import minimize\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport re\nfrom colorama import Fore, Style\n\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nimport lightgbm as lgb\nfrom catboost import CatBoostRegressor, CatBoostClassifier\nfrom xgboost import XGBRegressor\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\n\nSEED = 42\nn_splits = 5\n\nimport numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n\n    # Convert 'time_of_day' to datetime\n    df['time_of_day'] = pd.to_datetime(df['time_of_day'])\n    df.set_index('time_of_day', inplace=True)\n\n    # Daily aggregation\n    daily_features = df.resample('D').agg({\n        'X': ['mean', 'std', 'max', 'min', 'median', 'skew', lambda x: kurtosis(x, nan_policy='omit')],\n        'Y': ['mean', 'std', 'max', 'min', 'median', 'skew', lambda x: kurtosis(x, nan_policy='omit')],\n        'Z': ['mean', 'std', 'max', 'min', 'median', 'skew', lambda x: kurtosis(x, nan_policy='omit')],\n        'enmo': ['mean', 'std', 'max', 'min', 'median', 'skew', lambda x: kurtosis(x, nan_policy='omit')],\n        'light': ['mean', 'std'],\n        'battery_voltage': ['mean', 'std'],\n        'non-wear_flag': 'sum'\n    })\n\n    # Flatten the MultiIndex columns\n    daily_features.columns = ['_'.join(col).strip() for col in daily_features.columns.values]\n    \n    # Add additional features\n    daily_features['activity_count'] = (df['enmo'] > 0.1).resample('D').sum()  # Adjust threshold as needed\n    daily_features['days_active'] = (df['non-wear_flag'] == 0).resample('D').sum()  # Count active days\n    \n    # Rate of change features\n    daily_features['enmo_change'] = daily_features['enmo_mean'].diff()\n    \n    # Rolling statistics\n    rolling_windows = [5, 10, 15]  # Days\n    for window in rolling_windows:\n        daily_features[f'enmo_rolling_mean_{window}'] = daily_features['enmo_mean'].rolling(window=window).mean()\n        daily_features[f'enmo_rolling_std_{window}'] = daily_features['enmo_std'].rolling(window=window).std()\n\n    # Frequency domain features using FFT\n    for axis in ['X', 'Y', 'Z', 'enmo']:\n        freq_features = np.fft.fft(df[axis])\n        daily_features[f'{axis}_dominant_freq'] = np.abs(freq_features).argmax()\n\n    # Reset index to retain 'id'\n    daily_features.reset_index(inplace=True)\n    \n    # Extract child ID from filename\n    child_id = filename.split('=')[1]\n    daily_features['id'] = child_id\n\n    return daily_features\n\n\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    # Concatenate all results into a single DataFrame\n    features_df = pd.concat(results, ignore_index=True)\n\n    return features_df\n\n# Build the autoencoder model\ndef build_autoencoder(input_dim, encoding_dim):\n    input_layer = Input(shape=(input_dim,))\n    \n    # Encoder: compressing the input\n    encoded = Dense(encoding_dim, activation='relu')(input_layer)\n    \n    # Decoder: reconstructing the input\n    decoded = Dense(input_dim, activation='sigmoid')(encoded)\n    \n    # Autoencoder model\n    autoencoder = Model(inputs=input_layer, outputs=decoded)\n    \n    # Encoder model (for getting the compressed representation)\n    encoder = Model(inputs=input_layer, outputs=encoded)\n    \n    autoencoder.compile(optimizer=Adam(), loss='mse')\n    \n    return autoencoder, encoder\n\n# Function to perform dimensionality reduction using an autoencoder\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n    \"\"\"\n    Perform dimensionality reduction using an Autoencoder.\n\n    Parameters:\n    df (pd.DataFrame): The input DataFrame with numerical features.\n    encoding_dim (int): The dimension of the encoded space.\n    epochs (int): Number of epochs to train the autoencoder.\n    batch_size (int): Size of the batches for training.\n\n    Returns:\n    pd.DataFrame: DataFrame containing the reduced-dimensional representation (encoding).\n    \"\"\"\n    \n    # Step 1: Standardize the data\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n    \n    # Step 2: Build the autoencoder\n    input_dim = df_scaled.shape[1]\n    autoencoder, encoder = build_autoencoder(input_dim, encoding_dim)\n    \n    # Step 3: Train the autoencoder\n    autoencoder.fit(df_scaled, df_scaled, epochs=epochs, batch_size=batch_size, shuffle=True, verbose=1)\n    \n    # Step 4: Get the encoded (reduced) representation\n    encoded_data = encoder.predict(df_scaled)\n    \n    # Step 5: Create a DataFrame for the encoded features\n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i+1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n\n# Load the data\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")","metadata":{"papermill":{"duration":529.031327,"end_time":"2024-10-16T01:17:28.824624","exception":false,"start_time":"2024-10-16T01:08:39.793297","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-10-27T04:10:20.278026Z","iopub.execute_input":"2024-10-27T04:10:20.278468Z"},"trusted":true},"outputs":[{"name":"stderr","text":"  3%|▎         | 33/996 [00:17<06:19,  2.54it/s]","output_type":"stream"}],"execution_count":null},{"cell_type":"code","source":"# Columns to drop\ncolumns_to_drop = [\n    'enmo_change',\n    'enmo_rolling_mean_5',\n    'enmo_rolling_std_5',\n    'enmo_rolling_mean_10',\n    'enmo_rolling_std_10',\n    'enmo_rolling_mean_15',\n    'enmo_rolling_std_15'\n]\n\n# Drop the columns\ntrain_ts.drop(columns=columns_to_drop, inplace=True)\n\n# Drop the columns\ntest_ts.drop(columns=columns_to_drop, inplace=True)\n","metadata":{"papermill":{"duration":0.087567,"end_time":"2024-10-16T01:17:28.982636","exception":false,"start_time":"2024-10-16T01:17:28.895069","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts","metadata":{"papermill":{"duration":0.13241,"end_time":"2024-10-16T01:17:29.183102","exception":false,"start_time":"2024-10-16T01:17:29.050692","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop 'id' column for training\ndf_train = train_ts.drop('id', axis=1)\ndf_test = test_ts.drop('id', axis=1)","metadata":{"papermill":{"duration":0.083267,"end_time":"2024-10-16T01:17:29.338866","exception":false,"start_time":"2024-10-16T01:17:29.255599","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.dtypes","metadata":{"papermill":{"duration":0.08524,"end_time":"2024-10-16T01:17:29.491747","exception":false,"start_time":"2024-10-16T01:17:29.406507","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train","metadata":{"papermill":{"duration":0.125655,"end_time":"2024-10-16T01:17:29.687851","exception":false,"start_time":"2024-10-16T01:17:29.562196","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import StandardScaler\nfrom keras.models import Model\nfrom keras.layers import Input, Dense\nfrom keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping\n\ndef build_autoencoder(input_dim, encoding_dim):\n    input_layer = Input(shape=(input_dim,))\n    \n    # Encoder: compressing the input\n    encoded = Dense(encoding_dim, activation='relu')(input_layer)\n    \n    # Decoder: reconstructing the input\n    decoded = Dense(input_dim, activation='sigmoid')(encoded)\n    \n    # Autoencoder model\n    autoencoder = Model(inputs=input_layer, outputs=decoded)\n    \n    # Encoder model (for getting the compressed representation)\n    encoder = Model(inputs=input_layer, outputs=encoded)\n    \n    autoencoder.compile(optimizer=Adam(), loss='mse')\n    \n    return autoencoder, encoder\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=200, batch_size=32):\n    scaler = StandardScaler()\n    \n    # Exclude non-numeric columns\n    df_numeric = df.select_dtypes(include=[np.number])\n    \n    # Check for and handle NaN values\n    if df_numeric.isnull().values.any():\n        df_numeric.fillna(df_numeric.mean(), inplace=True)\n\n    if np.isinf(df_numeric).values.any():\n        df_numeric.replace([np.inf, -np.inf], np.nan, inplace=True)\n        df_numeric.fillna(df_numeric.mean(), inplace=True)\n\n    df_scaled = scaler.fit_transform(df_numeric)\n\n    # Check for NaN values after scaling\n    if np.isnan(df_scaled).any() or np.isinf(df_scaled).any():\n        print(\"NaN or infinite values detected in the scaled data!\")\n        return None  # or handle as needed\n\n    input_dim = df_scaled.shape[1]\n    autoencoder, encoder = build_autoencoder(input_dim, encoding_dim)\n    \n    # Use a smaller learning rate for stability\n    autoencoder.compile(optimizer=Adam(learning_rate=0.0001), loss='mse')\n    \n    # Set up early stopping\n    early_stopping = EarlyStopping(monitor='loss', patience=10, restore_best_weights=True)\n\n    # Train the autoencoder with early stopping\n    autoencoder.fit(df_scaled, df_scaled, \n                    epochs=epochs, \n                    batch_size=batch_size, \n                    shuffle=True, \n                    verbose=1, \n                    callbacks=[early_stopping])\n\n    encoded_data = encoder.predict(df_scaled)\n    \n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i+1}' for i in range(encoded_data.shape[1])])\n    \n    return df_encoded\n","metadata":{"papermill":{"duration":16.400711,"end_time":"2024-10-16T01:17:46.160059","exception":false,"start_time":"2024-10-16T01:17:29.759348","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Perform autoencoder dimensionality reduction\n# train_ts_encoded = perform_autoencoder(df_train, encoding_dim=60, epochs=100, batch_size=32)\n# test_ts_encoded = perform_autoencoder(df_test, encoding_dim=60, epochs=100, batch_size=32)\n\ntrain_ts_encoded = perform_autoencoder(df_train, encoding_dim=64, epochs=256, batch_size=32)\ntest_ts_encoded = perform_autoencoder(df_test, encoding_dim=64,epochs=256, batch_size=32)\n","metadata":{"papermill":{"duration":35.604464,"end_time":"2024-10-16T01:18:21.834548","exception":false,"start_time":"2024-10-16T01:17:46.230084","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts_encoded","metadata":{"papermill":{"duration":0.308849,"end_time":"2024-10-16T01:18:22.375663","exception":false,"start_time":"2024-10-16T01:18:22.066814","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ts_encoded","metadata":{"papermill":{"duration":0.31888,"end_time":"2024-10-16T01:18:22.928964","exception":false,"start_time":"2024-10-16T01:18:22.610084","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts_encoded","metadata":{"papermill":{"duration":0.31544,"end_time":"2024-10-16T01:18:23.484552","exception":false,"start_time":"2024-10-16T01:18:23.169112","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"time_series_cols = train_ts_encoded.columns.tolist()\n#time_series_cols.remove(\"id\")","metadata":{"papermill":{"duration":0.244567,"end_time":"2024-10-16T01:18:23.961403","exception":false,"start_time":"2024-10-16T01:18:23.716836","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts_encoded[\"id\"]=train_ts[\"id\"]\n\ntest_ts_encoded['id']=test_ts[\"id\"]\n#test_ts_pca[\"id\"]=test_ts[\"id\"]","metadata":{"papermill":{"duration":0.241313,"end_time":"2024-10-16T01:18:24.436402","exception":false,"start_time":"2024-10-16T01:18:24.195089","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ts_encoded","metadata":{"papermill":{"duration":0.323537,"end_time":"2024-10-16T01:18:24.995631","exception":false,"start_time":"2024-10-16T01:18:24.672094","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#train = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts_encoded, how=\"left\", on='id')\n#test_ = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = pd.merge(train, train_ts_encoded, how=\"left\", on='id')\n#test = pd.merge(test, test_ts, how=\"inner\", on='id')","metadata":{"papermill":{"duration":0.274048,"end_time":"2024-10-16T01:18:25.506521","exception":false,"start_time":"2024-10-16T01:18:25.232473","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.shape","metadata":{"papermill":{"duration":0.249054,"end_time":"2024-10-16T01:18:26.047596","exception":false,"start_time":"2024-10-16T01:18:25.798542","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.shape","metadata":{"papermill":{"duration":0.246574,"end_time":"2024-10-16T01:18:26.529568","exception":false,"start_time":"2024-10-16T01:18:26.282994","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"analysis=train","metadata":{"papermill":{"duration":0.245691,"end_time":"2024-10-16T01:18:27.013373","exception":false,"start_time":"2024-10-16T01:18:26.767682","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"analysis = analysis.dropna(subset='Enc_4')","metadata":{"papermill":{"duration":0.2491,"end_time":"2024-10-16T01:18:27.498662","exception":false,"start_time":"2024-10-16T01:18:27.249562","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"analysis=analysis.dropna(subset='sii')","metadata":{"papermill":{"duration":0.245138,"end_time":"2024-10-16T01:18:27.981566","exception":false,"start_time":"2024-10-16T01:18:27.736428","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"analysis = analysis.drop('id', axis=1)","metadata":{"papermill":{"duration":0.245555,"end_time":"2024-10-16T01:18:28.464618","exception":false,"start_time":"2024-10-16T01:18:28.219063","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def engineer_features(df):\n    # Create interaction features\n    df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    \n    # Create categorical features\n    #df['Age_Group'] = pd.cut(df['Basic_Demos-Age'], bins=[0, 12, 18, 25, 100], labels=['Child', 'Teen', 'Young Adult', 'Adult'])\n    #df['BMI_Category'] = pd.cut(df['Physical-BMI'], bins=[0, 18.5, 25, 30, 100], labels=['Underweight', 'Normal', 'Overweight', 'Obese'])\n    \n    return df\n\ntest = engineer_features(test)","metadata":{"papermill":{"duration":0.248911,"end_time":"2024-10-16T01:18:29.012551","exception":false,"start_time":"2024-10-16T01:18:28.76364","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.shape","metadata":{"papermill":{"duration":0.246176,"end_time":"2024-10-16T01:18:29.492962","exception":false,"start_time":"2024-10-16T01:18:29.246786","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train=engineer_features(train)","metadata":{"papermill":{"duration":0.252,"end_time":"2024-10-16T01:18:29.981021","exception":false,"start_time":"2024-10-16T01:18:29.729021","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.shape","metadata":{"papermill":{"duration":0.246652,"end_time":"2024-10-16T01:18:30.464766","exception":false,"start_time":"2024-10-16T01:18:30.218114","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#train = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)","metadata":{"papermill":{"duration":0.248613,"end_time":"2024-10-16T01:18:30.951172","exception":false,"start_time":"2024-10-16T01:18:30.702559","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nan_rows = train.query('sii.isna()', engine='python')","metadata":{"papermill":{"duration":0.287418,"end_time":"2024-10-16T01:18:31.475065","exception":false,"start_time":"2024-10-16T01:18:31.187647","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train.isna().sum().sum()-train_imputed.isna().sum().sum()","metadata":{"papermill":{"duration":0.242642,"end_time":"2024-10-16T01:18:31.950489","exception":false,"start_time":"2024-10-16T01:18:31.707847","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.impute import KNNImputer\n\n# Assuming 'train' is your DataFrame\n\n# Step 1: Create the KNN imputer\nimputer = KNNImputer(n_neighbors=5)  # You can adjust the number of neighbors\n\n# Step 2: Select numeric columns and fit the imputer\nnumeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\nimputed_data = imputer.fit_transform(train[numeric_cols])\n\n# Step 3: Create a new DataFrame with the imputed values\ntrain_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n\n# Step 4: Convert the 'sii' column back to integers\ntrain_imputed['sii'] = train_imputed['sii'].round().astype(int)\n\n# If there are other columns to retain, you can merge them back\nfor col in train.columns:\n    if col not in numeric_cols:\n        train_imputed[col] = train[col]\n\n# Now, check if 'sii' has been filled and is of integer type\nprint(train_imputed['sii'].isna().sum())  # Should be 0 if all NaNs were filled\nprint(train_imputed['sii'].dtype)  # Should show 'int'\n\ntrain_imputed['sii']=train['sii']\n\ntrain=train_imputed","metadata":{"papermill":{"duration":9.834466,"end_time":"2024-10-16T01:18:42.019744","exception":false,"start_time":"2024-10-16T01:18:32.185278","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii','BMI_Age','Internet_Hours_Age','BMI_Internet_Hours']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\n#test= test[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', \n          'FGC-Season', 'BIA-Season', 'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df","metadata":{"papermill":{"duration":0.323965,"end_time":"2024-10-16T01:18:42.578795","exception":false,"start_time":"2024-10-16T01:18:42.25483","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"featuresCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday','BMI_Age','Internet_Hours_Age','BMI_Internet_Hours']\n\nfeaturesCols += time_series_cols\n","metadata":{"papermill":{"duration":0.269703,"end_time":"2024-10-16T01:18:43.097256","exception":false,"start_time":"2024-10-16T01:18:42.827553","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test= test[featuresCols]","metadata":{"papermill":{"duration":0.249834,"end_time":"2024-10-16T01:18:43.586147","exception":false,"start_time":"2024-10-16T01:18:43.336313","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# #train['Age_Group'] = train['Age_Group'].cat.add_categories(['Missing'])\n\n# # Step 2: Fill missing values with 'Missing'\n# #train['Age_Group'] = train['Age_Group'].fillna('Missing')\n\n#train['BMI_Category'] = train['BMI_Category'].cat.add_categories(['Missing'])\n\n# # Step 2: Fill missing values with 'Missing'\n#train['BMI_Category'] = train['BMI_Category'].fillna('Missing')","metadata":{"papermill":{"duration":0.244868,"end_time":"2024-10-16T01:18:44.066745","exception":false,"start_time":"2024-10-16T01:18:43.821877","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# #test['Age_Group'] = test['Age_Group'].cat.add_categories(['Missing'])\n\n# # Step 2: Fill missing values with 'Missing'\n# #test['Age_Group'] = test['Age_Group'].fillna('Missing')\n\n#test['BMI_Category'] = test['BMI_Category'].cat.add_categories(['Missing'])\n\n# # Step 2: Fill missing values with 'Missing'\n#test['BMI_Category'] = test['BMI_Category'].fillna('Missing')","metadata":{"papermill":{"duration":0.246725,"end_time":"2024-10-16T01:18:44.547688","exception":false,"start_time":"2024-10-16T01:18:44.300963","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test","metadata":{"papermill":{"duration":0.470339,"end_time":"2024-10-16T01:18:45.252298","exception":false,"start_time":"2024-10-16T01:18:44.781959","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# #train = train.drop('BMI_Category', axis=1)\n# test = test.drop('BMI_Category', axis=1)","metadata":{"papermill":{"duration":0.250846,"end_time":"2024-10-16T01:18:45.743646","exception":false,"start_time":"2024-10-16T01:18:45.4928","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}","metadata":{"papermill":{"duration":0.340734,"end_time":"2024-10-16T01:18:46.324617","exception":false,"start_time":"2024-10-16T01:18:45.983883","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"This Mapping Works Fine For me I also Check Each Values in Train and test Using Logic. There no Data Lekage.\"\"\"\n\nfor col in cat_c:\n    mapping_train = create_mapping(col, train)\n    mapping_test = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping_train).astype(int)\n    test[col] = test[col].replace(mapping_test).astype(int)\n\nprint(f'Train Shape : {train.shape} || Test Shape : {test.shape}')","metadata":{"papermill":{"duration":0.299425,"end_time":"2024-10-16T01:18:46.865841","exception":false,"start_time":"2024-10-16T01:18:46.566416","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ntrain.head()","metadata":{"papermill":{"duration":0.368789,"end_time":"2024-10-16T01:18:47.47369","exception":false,"start_time":"2024-10-16T01:18:47.104901","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ntest.head()","metadata":{"papermill":{"duration":0.383055,"end_time":"2024-10-16T01:18:48.100195","exception":false,"start_time":"2024-10-16T01:18:47.71714","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.sii.unique()","metadata":{"papermill":{"duration":0.261251,"end_time":"2024-10-16T01:18:48.605621","exception":false,"start_time":"2024-10-16T01:18:48.34437","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Find columns missing in df2 but present in df1\nset(train.columns) - set(test.columns)","metadata":{"papermill":{"duration":0.260913,"end_time":"2024-10-16T01:18:49.113171","exception":false,"start_time":"2024-10-16T01:18:48.852258","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"analysis=train.dropna(subset='Enc_51')","metadata":{"papermill":{"duration":0.261936,"end_time":"2024-10-16T01:18:49.621953","exception":false,"start_time":"2024-10-16T01:18:49.360017","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_df=train[train['Enc_51'].isna()]","metadata":{"papermill":{"duration":0.260913,"end_time":"2024-10-16T01:18:50.192051","exception":false,"start_time":"2024-10-16T01:18:49.931138","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_df","metadata":{"papermill":{"duration":0.405512,"end_time":"2024-10-16T01:18:50.839957","exception":false,"start_time":"2024-10-16T01:18:50.434445","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Assuming analysis is your complete DataFrame (1000 data points)\n# and missing_df is your DataFrame with missing values (3000 data points)\n\n# Step 1: Store original data types\noriginal_dtypes = analysis.dtypes.to_dict()\n\n# Step 2: Split the complete data into features and target\nX_complete = analysis.drop('sii', axis=1)\ny_complete = analysis['sii']\n\n# Step 3: Identify categorical and numerical columns\ncategorical_features = [\n    'Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n    'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n    'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season'\n]\n\n# Step 3.1: Drop categorical columns for mean calculation\nnumerical_df = analysis.drop(columns=categorical_features + ['sii'])\n\n# Group by the target variable and calculate means for numerical features\nmean_impute = numerical_df.groupby(analysis['sii']).mean()\n\n# Group by the target variable and calculate modes for categorical features\nmode_impute = analysis[categorical_features].groupby(analysis['sii']).agg(\n    lambda x: x.mode()[0] if not x.mode().empty else None\n)\n\n# Step 4: Fill missing values class-wise\ndef fill_missing_values(row):\n    class_value = row['sii']\n    if pd.isnull(class_value):\n        return row  # No action if class_value is NaN\n\n    for col in row.index:\n        if pd.isnull(row[col]):\n            if col in mean_impute.columns and class_value in mean_impute.index:  # For numerical columns\n                row[col] = mean_impute.loc[class_value, col]\n            elif col in mode_impute.columns and class_value in mode_impute.index:  # For categorical columns\n                row[col] = mode_impute.loc[class_value, col]\n    return row\n\n# Apply the filling function to each row in the missing_df\ndf_missing_filled = missing_df.apply(fill_missing_values, axis=1)\n\n# Step 5: Combine datasets\nfinal_dataset = pd.concat([analysis, df_missing_filled], ignore_index=True)\n\n# Step 6: Convert categorical features to int64 safely\nfor col in categorical_features:\n    # Check for unique values and their counts\n    unique_values = final_dataset[col].unique()\n    print(f'Unique values in {col}: {unique_values}')\n\n    # Fill NaNs with a placeholder (optional)\n    final_dataset[col] = final_dataset[col].fillna('missing_value')\n\n    # Convert to category type first to manage the encoding\n    final_dataset[col] = final_dataset[col].astype('category')\n    \n    # Factorize the column\n    final_dataset[col] = pd.factorize(final_dataset[col])[0].astype('int64')\n\n# Step 7: Restore original data types for other columns\nfinal_dataset = final_dataset.astype(original_dtypes)\n\n# Now, final_dataset contains all your data with missing values filled and original data types restored\n","metadata":{"papermill":{"duration":8.319234,"end_time":"2024-10-16T01:18:59.407552","exception":false,"start_time":"2024-10-16T01:18:51.088318","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mean_impute.columns","metadata":{"papermill":{"duration":0.260074,"end_time":"2024-10-16T01:18:59.9163","exception":false,"start_time":"2024-10-16T01:18:59.656226","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mode_impute.columns","metadata":{"papermill":{"duration":0.2574,"end_time":"2024-10-16T01:19:00.422228","exception":false,"start_time":"2024-10-16T01:19:00.164828","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train=final_dataset","metadata":{"papermill":{"duration":0.258235,"end_time":"2024-10-16T01:19:00.929761","exception":false,"start_time":"2024-10-16T01:19:00.671526","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_df['FGC-Season']","metadata":{"papermill":{"duration":0.262073,"end_time":"2024-10-16T01:19:01.443303","exception":false,"start_time":"2024-10-16T01:19:01.18123","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.ensemble import VotingRegressor\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import cohen_kappa_score\nfrom scipy.optimize import minimize\nfrom tqdm import tqdm\nfrom IPython.display import clear_output\n\n# Function definitions (unchanged)\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    # Ensure all categorical columns are encoded\n    for col in categorical_features:\n        X[col] = pd.factorize(X[col])[0].astype('int64')\n    \n    test_data = test_data.copy()\n    for col in categorical_features:\n        test_data[col] = pd.factorize(test_data[col])[0].astype('int64')\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {tKappa:.3f}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission\n\n# Model parameters and instantiation (unchanged)\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,\n    'lambda_l2': 0.01\n}\n\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,\n    'reg_lambda': 5,\n    'random_state': SEED\n}\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,\n    'verbose': 0,\n    'l2_leaf_reg': 10\n}\n\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n])\n\n# Train the ensemble model\nSubmission = TrainML(voting_model, test)\n\n# Save submission\n# Submission.to_csv('submission.csv', index=False)\n","metadata":{"papermill":{"duration":53.335039,"end_time":"2024-10-16T01:19:55.090628","exception":false,"start_time":"2024-10-16T01:19:01.755589","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.shape","metadata":{"papermill":{"duration":0.292787,"end_time":"2024-10-16T01:19:55.641456","exception":false,"start_time":"2024-10-16T01:19:55.348669","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nSEED = 42\nn_splits = 5\n\n# Load datasets\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n\n\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\n\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    \n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    \n    stats, indexes = zip(*results)\n    \n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\n        \ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)   \n\n\n##########\n\n# import pandas as pd\n# from sklearn.impute import KNNImputer\n\n# # Assuming 'train' is your DataFrame\n\n# # Step 1: Create the KNN imputer\n# imputer = KNNImputer(n_neighbors=5)  # You can adjust the number of neighbors\n\n# # Step 2: Select numeric columns and fit the imputer\n# numeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\n# imputed_data = imputer.fit_transform(train[numeric_cols])\n\n# # Step 3: Create a new DataFrame with the imputed values\n# train_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n\n# # Step 4: Convert the 'sii' column back to integers\n# train_imputed['sii'] = train_imputed['sii'].round().astype(int)\n\n# # If there are other columns to retain, you can merge them back\n# for col in train.columns:\n#     if col not in numeric_cols:\n#         train_imputed[col] = train[col]\n\n# # Now, check if 'sii' has been filled and is of integer type\n# print(train_imputed['sii'].isna().sum())  # Should be 0 if all NaNs were filled\n# print(train_imputed['sii'].dtype)  # Should show 'int'\n\n# train_imputed['sii']=train['sii']\n\n# train=train_imputed\n\n\n#########\n\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex',\n                'CGAS-Season', 'CGAS-CGAS_Score', 'Physical-Season', 'Physical-BMI',\n                'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n                'Fitness_Endurance-Season', 'Fitness_Endurance-Max_Stage',\n                'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n                'FGC-Season', 'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND',\n                'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU',\n                'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR',\n                'FGC-FGC_SRR_Zone', 'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season',\n                'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM',\n                'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat', 'BIA-BIA_Frame_num',\n                'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM',\n                'BIA-BIA_TBW', 'PAQ_A-Season', 'PAQ_A-PAQ_A_Total', 'PAQ_C-Season',\n                'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw',\n                'SDS-SDS_Total_T', 'PreInt_EduHx-Season',\n                'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', \n          'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n          'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n        \ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tpTuned = threshold_Rounder(tpm, KappaOPtimizer.x)\n    \n    submission = pd.DataFrame({\n        'id': sample['id'],\n        'sii': tpTuned\n    })\n\n    return submission\n\n# Model parameters for LightGBM\nParams = {\n    'learning_rate': 0.046,\n    'max_depth': 12,\n    'num_leaves': 478,\n    'min_data_in_leaf': 13,\n    'feature_fraction': 0.893,\n    'bagging_fraction': 0.784,\n    'bagging_freq': 4,\n    'lambda_l1': 10,  # Increased from 6.59\n    'lambda_l2': 0.01  # Increased from 2.68e-06\n}\n\n\n# XGBoost parameters\nXGB_Params = {\n    'learning_rate': 0.05,\n    'max_depth': 6,\n    'n_estimators': 200,\n    'subsample': 0.8,\n    'colsample_bytree': 0.8,\n    'reg_alpha': 1,  # Increased from 0.1\n    'reg_lambda': 5,  # Increased from 1\n    'random_state': SEED\n}\n\n\nCatBoost_Params = {\n    'learning_rate': 0.05,\n    'depth': 6,\n    'iterations': 200,\n    'random_seed': SEED,\n    'cat_features': cat_c,\n    'verbose': 0,\n    'l2_leaf_reg': 10  # Increase this value\n}\n\n# Create model instances\nLight = LGBMRegressor(**Params, random_state=SEED, verbose=-1, n_estimators=300)\nXGB_Model = XGBRegressor(**XGB_Params)\nCatBoost_Model = CatBoostRegressor(**CatBoost_Params)\n\n# Combine models using Voting Regressor\nvoting_model = VotingRegressor(estimators=[\n    ('lightgbm', Light),\n    ('xgboost', XGB_Model),\n    ('catboost', CatBoost_Model)\n])\n\n# Train the ensemble model\nSubmission1 = TrainML(voting_model, test)\n\n# Save submission\n#Submission1.to_csv('submission.csv', index=False)\n","metadata":{"papermill":{"duration":249.381649,"end_time":"2024-10-16T01:24:05.295075","exception":false,"start_time":"2024-10-16T01:19:55.913426","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Submission1","metadata":{"papermill":{"duration":0.264925,"end_time":"2024-10-16T01:24:05.869189","exception":false,"start_time":"2024-10-16T01:24:05.604264","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.247397,"end_time":"2024-10-16T01:24:06.366487","exception":false,"start_time":"2024-10-16T01:24:06.11909","status":"completed"},"tags":[]},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\nfrom sklearn.base import clone\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom scipy.optimize import minimize\nfrom concurrent.futures import ThreadPoolExecutor\nfrom tqdm import tqdm\n\nfrom colorama import Fore, Style\nfrom IPython.display import clear_output\nimport warnings\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.ensemble import VotingRegressor, RandomForestRegressor, GradientBoostingRegressor\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nSEED = 42\nn_splits = 5\n\n# Load datasets\ntrain = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv')\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\ndef process_file(filename, dirname):\n    df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    df.drop('step', axis=1, inplace=True)\n    return df.describe().values.reshape(-1), filename.split('=')[1]\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n    stats, indexes = zip(*results)\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\ntime_series_cols = train_ts.columns.tolist()\ntime_series_cols.remove(\"id\")\n\ntrain = pd.merge(train, train_ts, how=\"left\", on='id')\ntest = pd.merge(test, test_ts, how=\"left\", on='id')\n\ntrain = train.drop('id', axis=1)\ntest = test.drop('id', axis=1)\n\n\n#####\n\n\n# import pandas as pd\n# from sklearn.impute import KNNImputer\n\n# # Assuming 'train' is your DataFrame\n\n# # Step 1: Create the KNN imputer\n# imputer = KNNImputer(n_neighbors=5)  # You can adjust the number of neighbors\n\n# # Step 2: Select numeric columns and fit the imputer\n# numeric_cols = train.select_dtypes(include=['float64', 'int64']).columns\n# imputed_data = imputer.fit_transform(train[numeric_cols])\n\n# # Step 3: Create a new DataFrame with the imputed values\n# train_imputed = pd.DataFrame(imputed_data, columns=numeric_cols)\n\n# # Step 4: Convert the 'sii' column back to integers\n# train_imputed['sii'] = train_imputed['sii'].round().astype(int)\n\n# # If there are other columns to retain, you can merge them back\n# for col in train.columns:\n#     if col not in numeric_cols:\n#         train_imputed[col] = train[col]\n\n# # Now, check if 'sii' has been filled and is of integer type\n# print(train_imputed['sii'].isna().sum())  # Should be 0 if all NaNs were filled\n# print(train_imputed['sii'].dtype)  # Should show 'int'\n\n# train_imputed['sii']=train['sii']\n\n# train=train_imputed\n\n#####\n\nfeaturesCols = ['Basic_Demos-Enroll_Season', 'Basic_Demos-Age', 'Basic_Demos-Sex', 'CGAS-Season', 'CGAS-CGAS_Score',\n                'Physical-Season', 'Physical-BMI', 'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n                'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP', 'Fitness_Endurance-Season',\n                'Fitness_Endurance-Max_Stage', 'Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec', 'FGC-Season',\n                'FGC-FGC_CU', 'FGC-FGC_CU_Zone', 'FGC-FGC_GSND', 'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD', 'FGC-FGC_GSD_Zone',\n                'FGC-FGC_PU', 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR', 'FGC-FGC_SRR_Zone',\n                'FGC-FGC_TL', 'FGC-FGC_TL_Zone', 'BIA-Season', 'BIA-BIA_Activity_Level_num', 'BIA-BIA_BMC', 'BIA-BIA_BMI',\n                'BIA-BIA_BMR', 'BIA-BIA_DEE', 'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI', 'BIA-BIA_Fat',\n                'BIA-BIA_Frame_num', 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST', 'BIA-BIA_SMM', 'BIA-BIA_TBW', 'PAQ_A-Season',\n                'PAQ_A-PAQ_A_Total', 'PAQ_C-Season', 'PAQ_C-PAQ_C_Total', 'SDS-Season', 'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T',\n                'PreInt_EduHx-Season', 'PreInt_EduHx-computerinternet_hoursday', 'sii']\n\nfeaturesCols += time_series_cols\n\ntrain = train[featuresCols]\ntrain = train.dropna(subset='sii')\n\ncat_c = ['Basic_Demos-Enroll_Season', 'CGAS-Season', 'Physical-Season', 'Fitness_Endurance-Season', 'FGC-Season', 'BIA-Season', \n         'PAQ_A-Season', 'PAQ_C-Season', 'SDS-Season', 'PreInt_EduHx-Season']\n\ndef update(df):\n    global cat_c\n    for c in cat_c: \n        df[c] = df[c].fillna('Missing')\n        df[c] = df[c].astype('category')\n    return df\n\ntrain = update(train)\ntest = update(test)\n\ndef create_mapping(column, dataset):\n    unique_values = dataset[column].unique()\n    return {value: idx for idx, value in enumerate(unique_values)}\n\nfor col in cat_c:\n    mapping = create_mapping(col, train)\n    mappingTe = create_mapping(col, test)\n    \n    train[col] = train[col].replace(mapping).astype(int)\n    test[col] = test[col].replace(mappingTe).astype(int)\n\ndef quadratic_weighted_kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\ndef threshold_Rounder(oof_non_rounded, thresholds):\n    return np.where(oof_non_rounded < thresholds[0], 0,\n                    np.where(oof_non_rounded < thresholds[1], 1,\n                             np.where(oof_non_rounded < thresholds[2], 2, 3)))\n\ndef evaluate_predictions(thresholds, y_true, oof_non_rounded):\n    rounded_p = threshold_Rounder(oof_non_rounded, thresholds)\n    return -quadratic_weighted_kappa(y_true, rounded_p)\n\ndef TrainML(model_class, test_data):\n    X = train.drop(['sii'], axis=1)\n    y = train['sii']\n\n    SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n    \n    train_S = []\n    test_S = []\n    \n    oof_non_rounded = np.zeros(len(y), dtype=float) \n    oof_rounded = np.zeros(len(y), dtype=int) \n    test_preds = np.zeros((len(test_data), n_splits))\n\n    for fold, (train_idx, test_idx) in enumerate(tqdm(SKF.split(X, y), desc=\"Training Folds\", total=n_splits)):\n        X_train, X_val = X.iloc[train_idx], X.iloc[test_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[test_idx]\n\n        model = clone(model_class)\n        model.fit(X_train, y_train)\n\n        y_train_pred = model.predict(X_train)\n        y_val_pred = model.predict(X_val)\n\n        oof_non_rounded[test_idx] = y_val_pred\n        y_val_pred_rounded = y_val_pred.round(0).astype(int)\n        oof_rounded[test_idx] = y_val_pred_rounded\n\n        train_kappa = quadratic_weighted_kappa(y_train, y_train_pred.round(0).astype(int))\n        val_kappa = quadratic_weighted_kappa(y_val, y_val_pred_rounded)\n\n        train_S.append(train_kappa)\n        test_S.append(val_kappa)\n        \n        test_preds[:, fold] = model.predict(test_data)\n        \n        print(f\"Fold {fold+1} - Train QWK: {train_kappa:.4f}, Validation QWK: {val_kappa:.4f}\")\n        clear_output(wait=True)\n\n    print(f\"Mean Train QWK --> {np.mean(train_S):.4f}\")\n    print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n    KappaOPtimizer = minimize(evaluate_predictions,\n                              x0=[0.5, 1.5, 2.5], args=(y, oof_non_rounded), \n                              method='Nelder-Mead')\n    assert KappaOPtimizer.success, \"Optimization did not converge.\"\n    \n    oof_tuned = threshold_Rounder(oof_non_rounded, KappaOPtimizer.x)\n    tKappa = quadratic_weighted_kappa(y, oof_tuned)\n\n    print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n    tpm = test_preds.mean(axis=1)\n    tp_rounded = threshold_Rounder(tpm, KappaOPtimizer.x)\n\n    return tp_rounded\n\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\n\n# Imputation step: Filling missing values with the median\nimputer = SimpleImputer(strategy='median')\n\n# Updating the ensemble to include the RandomForest and GradientBoosting models\nensemble = VotingRegressor(estimators=[\n    ('lgb', Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])),\n    ('xgb', Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])),\n    ('cat', Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])),\n    ('rf', Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])),\n    ('gb', Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))]))\n])\n\n# Train the ensemble with the updated model pipeline\npredictions = TrainML(ensemble, test)\n\n# Save predictions to a CSV file\nsample['sii'] = predictions\n#sample.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":312.364146,"end_time":"2024-10-16T01:29:18.976347","exception":false,"start_time":"2024-10-16T01:24:06.612201","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample","metadata":{"papermill":{"duration":0.266453,"end_time":"2024-10-16T01:29:19.493911","exception":false,"start_time":"2024-10-16T01:29:19.227458","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\n# Load your three submission files\nsub1 = Submission\nsub2 = Submission1\nsub3 = sample\n\n# Ensure the IDs are aligned (if not sorted)\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\n# Combine the three predictions\ncombined = pd.DataFrame({\n    'id': sub1['id'],\n    'sii_1': sub1['sii'],\n    'sii_2': sub2['sii'],\n    'sii_3': sub3['sii']\n})\n\n# Apply majority voting\ndef majority_vote(row):\n    return row.mode()[0]  # Mode gets the most frequent value (majority vote)\n\ncombined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n\n# Create the final submission DataFrame\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\n# Save the final submission to a CSV file\nfinal_submission.to_csv('submission.csv', index=False)\n\nprint(\"Majority voting completed and saved to 'Final_Submission.csv'\")","metadata":{"papermill":{"duration":0.277622,"end_time":"2024-10-16T01:29:20.018389","exception":false,"start_time":"2024-10-16T01:29:19.740767","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Load your three submission files\n# sub1 = Submission\n# sub2 = Submission1\n# sub3 = sample\n\n# sub1 = sub1.sort_values(by='id').reset_index(drop=True)\n# sub2 = sub2.sort_values(by='id').reset_index(drop=True)\n# sub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\n# sub1 = sub1.rename(columns={'sii': 'sii_1'})\n# sub2 = sub2.rename(columns={'sii': 'sii_2'})\n# sub3 = sub3.rename(columns={'sii': 'sii_3'})\n# subs = pd.merge(sub1,sub2,on=['id'])\n# subs = pd.merge(subs,sub3,on=['id'])\n\n# #subs['sii_s'] = subs['sii_1'] *0.555 + 0.001* subs['sii_2'] + 0.444* subs['sii_3']\n# #subs['sii_s'] = subs['sii_1'] *0.450 + 0.450* subs['sii_2'] + 0.100* subs['sii_3']\n# #subs['sii_s'] = subs['sii_1'] *0.525 + 0.050* subs['sii_2'] + 0.425* subs['sii_3']\n# #subs['sii_s'] = subs['sii_1'] *0.500 + 0.100* subs['sii_2'] + 0.400* subs['sii_3']\n\n# subs['sii_s'] = np.round(subs['sii_1'] *0.85 + 0.10* subs['sii_2'] + 0.05* subs['sii_3'])\n\n# subs['sii_s'] = subs['sii_s'].astype(int)\n\n# combined = pd.DataFrame({\n    \n#     'id'   : sub1['id'],\n    \n#     'sii_1': sub1['sii_1'],\n#     'sii_2': sub2['sii_2'],\n#     'sii_3': sub3['sii_3'],\n    \n#     'sii_s': subs['sii_s'],\n# })\n\n# display(combined)\n\n# def majority_vote(row):\n#     return row.mode()[0]\n                                                         \n# combined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3', 'sii_s']].apply(majority_vote, axis=1)\n\n# final_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\n# final_submission.to_csv('submission.csv', index=False)\n\n# print(\"Majority voting completed and saved to 'Final_Submission.csv'\")","metadata":{"papermill":{"duration":0.261198,"end_time":"2024-10-16T01:29:20.533686","exception":false,"start_time":"2024-10-16T01:29:20.272488","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# X = train.drop(['sii'], axis=1)\n# y = train['sii']","metadata":{"papermill":{"duration":0.262815,"end_time":"2024-10-16T01:29:21.046072","exception":false,"start_time":"2024-10-16T01:29:20.783257","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# X_train=X\n# y_train=y","metadata":{"papermill":{"duration":0.263534,"end_time":"2024-10-16T01:29:21.562687","exception":false,"start_time":"2024-10-16T01:29:21.299153","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.model_selection import RandomizedSearchCV\n# from sklearn.impute import SimpleImputer\n# from sklearn.pipeline import Pipeline\n# from sklearn.ensemble import VotingRegressor\n# from lightgbm import LGBMRegressor\n# from xgboost import XGBRegressor\n# from catboost import CatBoostRegressor\n# from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n\n# # Define hyperparameter grids for each model\n\n# # LightGBM hyperparameter grid\n# lgb_param_grid = {\n#     'regressor__learning_rate': [0.01, 0.03, 0.046, 0.1],\n#     'regressor__max_depth': [8, 12, 16],\n#     'regressor__num_leaves': [128, 256, 478, 512],\n#     'regressor__min_data_in_leaf': [10, 13, 16],\n#     'regressor__feature_fraction': [0.7, 0.8, 0.893],\n#     'regressor__bagging_fraction': [0.7, 0.784, 0.85],\n#     'regressor__lambda_l1': [5, 10, 15],\n#     'regressor__lambda_l2': [0.001, 0.01, 0.1]\n# }\n\n# # XGBoost hyperparameter grid\n# xgb_param_grid = {\n#     'regressor__learning_rate': [0.01, 0.05, 0.1],\n#     'regressor__max_depth': [4, 6, 8],\n#     'regressor__n_estimators': [100, 200, 300],\n#     'regressor__subsample': [0.7, 0.8, 0.9],\n#     'regressor__colsample_bytree': [0.7, 0.8, 0.9],\n#     'regressor__reg_alpha': [0.1, 1, 5],\n#     'regressor__reg_lambda': [1, 5, 10]\n# }\n\n# # CatBoost hyperparameter grid\n# cat_param_grid = {\n#     'regressor__learning_rate': [0.03, 0.05, 0.07],\n#     'regressor__depth': [4, 6, 8],\n#     'regressor__iterations': [200, 300, 400],\n#     'regressor__l2_leaf_reg': [3, 10, 15]\n# }\n\n# # RandomForest hyperparameter grid\n# rf_param_grid = {\n#     'regressor__n_estimators': [100, 200, 300],\n#     'regressor__max_depth': [10, 20, 30],\n#     'regressor__min_samples_split': [2, 5, 10],\n#     'regressor__min_samples_leaf': [1, 2, 4]\n# }\n\n# # GradientBoosting hyperparameter grid\n# gb_param_grid = {\n#     'regressor__learning_rate': [0.01, 0.05, 0.1],\n#     'regressor__n_estimators': [100, 200, 300],\n#     'regressor__max_depth': [3, 5, 7],\n#     'regressor__subsample': [0.7, 0.8, 0.9]\n# }\n\n# # Imputer for handling missing values\n# imputer = SimpleImputer(strategy='median')\n\n# # Pipelines for each regressor\n# lgb_pipeline = Pipeline(steps=[('imputer', imputer), ('regressor', LGBMRegressor(random_state=SEED))])\n# xgb_pipeline = Pipeline(steps=[('imputer', imputer), ('regressor', XGBRegressor(random_state=SEED))])\n# cat_pipeline = Pipeline(steps=[('imputer', imputer), ('regressor', CatBoostRegressor(random_state=SEED, silent=True))])\n# rf_pipeline = Pipeline(steps=[('imputer', imputer), ('regressor', RandomForestRegressor(random_state=SEED))])\n# gb_pipeline = Pipeline(steps=[('imputer', imputer), ('regressor', GradientBoostingRegressor(random_state=SEED))])\n\n# # Perform RandomizedSearchCV for each model\n\n# # LightGBM\n# lgb_search = RandomizedSearchCV(lgb_pipeline, lgb_param_grid, n_iter=20, scoring='neg_mean_squared_error', cv=3, random_state=SEED)\n# lgb_search.fit(X_train, y_train)\n\n# # XGBoost\n# xgb_search = RandomizedSearchCV(xgb_pipeline, xgb_param_grid, n_iter=20, scoring='neg_mean_squared_error', cv=3, random_state=SEED)\n# xgb_search.fit(X_train, y_train)\n\n# # CatBoost\n# cat_search = RandomizedSearchCV(cat_pipeline, cat_param_grid, n_iter=20, scoring='neg_mean_squared_error', cv=3, random_state=SEED)\n# cat_search.fit(X_train, y_train)\n\n# # RandomForest\n# rf_search = RandomizedSearchCV(rf_pipeline, rf_param_grid, n_iter=20, scoring='neg_mean_squared_error', cv=3, random_state=SEED)\n# rf_search.fit(X_train, y_train)\n\n# # GradientBoosting\n# gb_search = RandomizedSearchCV(gb_pipeline, gb_param_grid, n_iter=20, scoring='neg_mean_squared_error', cv=3, random_state=SEED)\n# gb_search.fit(X_train, y_train)\n\n# # Get the best models from the RandomizedSearchCV results\n# best_lgb = lgb_search.best_estimator_\n# best_xgb = xgb_search.best_estimator_\n# best_cat = cat_search.best_estimator_\n# best_rf = rf_search.best_estimator_\n# best_gb = gb_search.best_estimator_\n\n# # Create a new ensemble with the best estimators\n# ensemble = VotingRegressor(estimators=[\n#     ('lgb', best_lgb),\n#     ('xgb', best_xgb),\n#     ('cat', best_cat),\n#     ('rf', best_rf),\n#     ('gb', best_gb)\n# ])\n\n# # Train the ensemble\n# predictions = TrainML(ensemble, test)\n\n# # Save predictions to a CSV file\n# sample['sii'] = predictions\n# sample.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":0.265233,"end_time":"2024-10-16T01:29:22.139391","exception":false,"start_time":"2024-10-16T01:29:21.874158","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.252327,"end_time":"2024-10-16T01:29:22.64439","exception":false,"start_time":"2024-10-16T01:29:22.392063","status":"completed"},"tags":[]},"outputs":[],"execution_count":null}]}