{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# LightGBM-DeepAutoEncoder Code\nThis code is for training LightGBM and was used for submission. It was originally run on Google Colaboratory and transferred into a notebook, so there might be some missing parts.","metadata":{}},{"cell_type":"code","source":"TARGET_COL = \"sii\"\n\nFEATURE_COLS = [\n    \"Basic_Demos-Enroll_Season\",\n    \"Basic_Demos-Age\",\n    \"Basic_Demos-Sex\",\n    \"CGAS-Season\",\n    \"CGAS-CGAS_Score\",\n    \"Physical-Season\",\n    \"Physical-BMI\",\n    \"Physical-Height\",\n    \"Physical-Weight\",\n    \"Physical-Waist_Circumference\",\n    \"Physical-Diastolic_BP\",\n    \"Physical-HeartRate\",\n    \"Physical-Systolic_BP\",\n    \"Fitness_Endurance-Season\",\n    \"Fitness_Endurance-Max_Stage\",\n    \"Fitness_Endurance-Time_Mins\",\n    \"Fitness_Endurance-Time_Sec\",\n    \"FGC-Season\",\n    \"FGC-FGC_CU\",\n    \"FGC-FGC_CU_Zone\",\n    \"FGC-FGC_GSND\",\n    \"FGC-FGC_GSND_Zone\",\n    \"FGC-FGC_GSD\",\n    \"FGC-FGC_GSD_Zone\",\n    \"FGC-FGC_PU\",\n    \"FGC-FGC_PU_Zone\",\n    \"FGC-FGC_SRL\",\n    \"FGC-FGC_SRL_Zone\",\n    \"FGC-FGC_SRR\",\n    \"FGC-FGC_SRR_Zone\",\n    \"FGC-FGC_TL\",\n    \"FGC-FGC_TL_Zone\",\n    \"BIA-Season\",\n    \"BIA-BIA_Activity_Level_num\",\n    \"BIA-BIA_BMC\",\n    \"BIA-BIA_BMI\",\n    \"BIA-BIA_BMR\",\n    \"BIA-BIA_DEE\",\n    \"BIA-BIA_ECW\",\n    \"BIA-BIA_FFM\",\n    \"BIA-BIA_FFMI\",\n    \"BIA-BIA_FMI\",\n    \"BIA-BIA_Fat\",\n    \"BIA-BIA_Frame_num\",\n    \"BIA-BIA_ICW\",\n    \"BIA-BIA_LDM\",\n    \"BIA-BIA_LST\",\n    \"BIA-BIA_SMM\",\n    \"BIA-BIA_TBW\",\n    \"PAQ_A-Season\",\n    \"PAQ_A-PAQ_A_Total\",\n    \"PAQ_C-Season\",\n    \"PAQ_C-PAQ_C_Total\",\n    \"SDS-Season\",\n    \"SDS-SDS_Total_Raw\",\n    \"SDS-SDS_Total_T\",\n    \"PreInt_EduHx-Season\",\n    \"PreInt_EduHx-computerinternet_hoursday\",\n]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering(df):\n    added_columns = []\n\n    normal_values = pd.DataFrame({\n        'Basic_Demos-Age': list(range(5, 23)) * 2,  \n        'Basic_Demos-Sex': [0] * 18 + [1] * 18,  \n        'Normal_BMI': [15 + i * 0.5 for i in range(18)] * 2,  \n        'Normal_BMR': [1100 + i * 50 for i in range(18)] * 2,  \n        'Normal_HeartRate': [80 for _ in range(36)],  \n        'Normal_Systolic_BP': [100 + i for i in range(18)] * 2,  \n        'Normal_Diastolic_BP': [65 + i for i in range(18)] * 2,  \n    })\n    df = df.merge(normal_values, on=['Basic_Demos-Age', 'Basic_Demos-Sex'], how='left')\n\n    df['cal_BMI_Inflation'] = df['Physical-BMI'] / df['Normal_BMI']  \n    added_columns.append('cal_BMI_Inflation')\n    df['cal_BMR_Inflation'] = df['BIA-BIA_BMR'] / df['Normal_BMR']  \n    added_columns.append('cal_BMR_Inflation')\n    df['cal_HeartRate_Inflation'] = df['Physical-HeartRate'] / df['Normal_HeartRate']  \n    added_columns.append('cal_HeartRate_Inflation')\n    df['cal_Systolic_BP_Inflation'] = df['Physical-Systolic_BP'] / df['Normal_Systolic_BP']  \n    added_columns.append('cal_Systolic_BP_Inflation')\n    df['cal_Diastolic_BP_Inflation'] = df['Physical-Diastolic_BP'] / df['Normal_Diastolic_BP']  \n    added_columns.append('cal_Diastolic_BP_Inflation')\n\n\n    season_start_month = {'Spring': 3, 'Summer': 6, 'Fall': 9, 'Winter': 12}\n    season_cols = [col for col in df.columns if 'Season' in col and 'PCIAT' not in col] \n    for col in season_cols:\n        df[col + '_StartMonth'] = df[col].map(season_start_month) \n        added_columns.append(col + '_StartMonth')\n        for season, start_month in season_start_month.items():\n            df[f'{col}_{season}'] = (df[col] == season).astype(float)  \n            added_columns.append(f'{col}_{season}')\n\n\n    if 'Basic_Demos-Enroll_Season' in season_cols:\n        for col in season_cols:\n            if col != 'Basic_Demos-Enroll_Season':\n                df[f'{col}_MonthDifference'] = df.apply(\n                    lambda row: (\n                        12 - abs(row['Basic_Demos-Enroll_Season_StartMonth'] - row[col + '_StartMonth'])\n                        if row['Basic_Demos-Enroll_Season_StartMonth'] > row[col + '_StartMonth'] else\n                        abs(row['Basic_Demos-Enroll_Season_StartMonth'] - row[col + '_StartMonth'])\n                    ) if pd.notna(row['Basic_Demos-Enroll_Season_StartMonth']) and pd.notna(row[col + '_StartMonth'])\n                    else np.nan, axis=1\n                )\n                added_columns.append(f'{col}_MonthDifference')\n\n    \n    df['cal_BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n    added_columns.append('cal_BMI_Age')\n    df['cal_Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n    added_columns.append('cal_Internet_Hours_Age')\n    df['cal_BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n    added_columns.append('cal_BMI_Internet_Hours')\n    df['cal_BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI'].replace(0, np.nan)\n    added_columns.append('cal_BFP_BMI')\n    df['cal_FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat'].replace(0, np.nan)\n    added_columns.append('cal_FFMI_BFP')\n    df['cal_FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat'].replace(0, np.nan)\n    added_columns.append('cal_FMI_BFP')\n    df['cal_LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW'].replace(0, np.nan)\n    added_columns.append('cal_LST_TBW')\n    df['cal_BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n    added_columns.append('cal_BFP_BMR')\n    df['cal_BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n    added_columns.append('cal_BFP_DEE')\n    df['cal_BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight'].replace(0, np.nan)\n    added_columns.append('cal_BMR_Weight')\n    df['cal_DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight'].replace(0, np.nan)\n    added_columns.append('cal_DEE_Weight')\n    df['cal_SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height'].replace(0, np.nan)\n    added_columns.append('cal_SMM_Height')\n    df['cal_Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI'].replace(0, np.nan)\n    added_columns.append('cal_Muscle_to_Fat')\n    df['cal_Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight'].replace(0, np.nan)\n    added_columns.append('cal_Hydration_Status')\n    df['cal_ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW'].replace(0, np.nan)\n    added_columns.append('cal_ICW_TBW')\n\n    df['cal_Sleep_Disturbance_Index'] = (df['SDS-SDS_Total_Raw'] * df['SDS-SDS_Total_T']) / 100\n    added_columns.append('cal_Sleep_Disturbance_Index')\n    df['cal_Strength_Flexibility_Score'] = (\n        df['FGC-FGC_GSD'] + df['FGC-FGC_GSND'] + df['FGC-FGC_SRL'] + df['FGC-FGC_SRR']\n    ) / 4\n    added_columns.append('cal_Strength_Flexibility_Score')\n    df['cal_Hydration_BMI'] = df['BIA-BIA_TBW'] / df['Physical-BMI']\n    added_columns.append('cal_Hydration_BMI')\n    df['cal_Metabolic_Risk'] = (\n        df['BIA-BIA_Fat'] + df['Physical-BMI'] + df['Physical-Systolic_BP'] + df['Physical-Diastolic_BP']\n    ) / 4\n    added_columns.append('cal_Metabolic_Risk')\n\n    return df, added_columns\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Time Series\ndef extract_advanced_features(data):\n    # Initial data preprocessing\n    data = data.copy()\n    data['timestamp'] = pd.to_datetime(data['relative_date_PCIAT'], unit='D') + pd.to_timedelta(data['time_of_day'])\n    data = data[data['non-wear_flag'] == 0]\n\n    # Calculate basic metrics\n    data['magnitude'] = np.sqrt(data['X']**2 + data['Y']**2 + data['Z']**2)\n    data['velocity'] = data['magnitude']\n    data['distance'] = data['velocity'] * 5  # 5 seconds per observation\n    data['date'] = data['timestamp'].dt.date\n    hour = pd.to_datetime(data['time_of_day']).dt.hour\n\n    # Calculate aggregated distances\n    distances = {\n        'daily': data.groupby('date')['distance'].sum(),\n        'monthly': data.groupby(data['timestamp'].dt.to_period('M'))['distance'].sum(),\n        'quarterly': data.groupby('quarter')['distance'].sum()\n    }\n\n    # Initialize features dictionary\n    features = {}\n\n    # Time masks for different periods\n    time_masks = {\n        'morning': (hour >= 6) & (hour < 12),\n        'afternoon': (hour >= 12) & (hour < 18),\n        'evening': (hour >= 18) & (hour < 22),\n        'night': (hour >= 22) | (hour < 6)\n    }\n\n    # 1. Activity Pattern Features\n    for period, mask in time_masks.items():\n        features.update({\n            f'{period}_activity_mean': data.loc[mask, 'magnitude'].mean(),\n            f'{period}_activity_std': data.loc[mask, 'magnitude'].std(),\n            f'{period}_enmo_mean': data.loc[mask, 'enmo'].mean()\n        })\n\n    # 2. Sleep Quality Features\n    sleep_hours = time_masks['night']\n    magnitude_threshold = data['magnitude'].mean() + data['magnitude'].std()\n\n    features.update({\n        'sleep_movement_mean': data.loc[sleep_hours, 'magnitude'].mean(),\n        'sleep_movement_std': data.loc[sleep_hours, 'magnitude'].std(),\n        'sleep_disruption_count': len(data.loc[sleep_hours & (data['magnitude'] >\n            data['magnitude'].mean() + 2 * data['magnitude'].std())]),\n        'light_exposure_during_sleep': data.loc[sleep_hours, 'light'].mean(),\n        'sleep_position_changes': len(data.loc[sleep_hours &\n            (abs(data['anglez'].diff()) > 45)]),\n        'good_sleep_cycle': int(data.loc[sleep_hours, 'light'].mean() < 50)\n    })\n\n    # 3. Activity Intensity Features\n    features.update({\n        'sedentary_time_ratio': (data['magnitude'] < magnitude_threshold * 0.5).mean(),\n        'moderate_activity_ratio': ((data['magnitude'] >= magnitude_threshold * 0.5) &\n            (data['magnitude'] < magnitude_threshold * 1.5)).mean(),\n        'vigorous_activity_ratio': (data['magnitude'] >= magnitude_threshold * 1.5).mean(),\n        'activity_peaks_per_day': len(data[data['magnitude'] >\n            data['magnitude'].quantile(0.95)]) / len(data.groupby('relative_date_PCIAT'))\n    })\n\n    # 4. Circadian Rhythm Features\n    hourly_activity = data.groupby(hour)['magnitude'].mean()\n    features.update({\n        'circadian_regularity': hourly_activity.std() / hourly_activity.mean(),\n        'peak_activity_hour': hourly_activity.idxmax(),\n        'trough_activity_hour': hourly_activity.idxmin(),\n        'activity_range': hourly_activity.max() - hourly_activity.min()\n    })\n\n    # 5-11. Additional Feature Groups\n    weekend_mask = data['weekday'].isin([6, 7])\n\n    features.update({\n        # Movement Patterns\n        'movement_entropy': stats.entropy(pd.qcut(data['magnitude'], q=10, duplicates='drop').value_counts()),\n        'direction_changes': len(data[abs(data['anglez'].diff()) > 30]) / len(data),\n        'sustained_activity_periods': len(data[data['magnitude'].rolling(12).mean() >\n            magnitude_threshold]) / len(data),\n\n        # Weekend vs Weekday\n        'weekend_activity_ratio': data.loc[weekend_mask, 'magnitude'].mean() /\n            data.loc[~weekend_mask, 'magnitude'].mean(),\n        'weekend_sleep_difference': data.loc[weekend_mask & sleep_hours, 'magnitude'].mean() -\n            data.loc[~weekend_mask & sleep_hours, 'magnitude'].mean(),\n\n        # Non-wear Time\n        'wear_time_ratio': (data['non-wear_flag'] == 0).mean(),\n        'wear_consistency': len(data['non-wear_flag'].value_counts()),\n        'longest_wear_streak': data['non-wear_flag'].eq(0).astype(int).groupby(\n            data['non-wear_flag'].ne(0).cumsum()).sum().max(),\n\n        # Device Usage\n        'screen_time_proxy': (data['light'] > data['light'].quantile(0.75)).mean(),\n        'dark_environment_ratio': (data['light'] < data['light'].quantile(0.25)).mean(),\n        'light_variation': data['light'].std() / data['light'].mean() if data['light'].mean() != 0 else 0,\n\n        # Battery Usage\n        'battery_drain_rate': -np.polyfit(range(len(data)), data['battery_voltage'], 1)[0],\n        'battery_variability': data['battery_voltage'].std(),\n        'low_battery_time': (data['battery_voltage'] < data['battery_voltage'].quantile(0.1)).mean(),\n\n        # Time-based\n        'days_monitored': data['relative_date_PCIAT'].nunique(),\n        'total_active_hours': len(data[data['magnitude'] > magnitude_threshold * 0.5]) * 5 / 3600,\n        'activity_regularity': data.groupby('weekday')['magnitude'].mean().std()\n    })\n\n    # Variability Features for multiple columns\n    for col in ['X', 'Y', 'Z', 'enmo', 'anglez']:\n        features.update({\n            f'{col}_skewness': data[col].skew(),\n            f'{col}_kurtosis': data[col].kurtosis(),\n            f'{col}_trend': np.polyfit(range(len(data)), data[col], 1)[0]\n        })\n\n    return pd.DataFrame([features])\n\ndef process_file(filename, dirname):\n    df= pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n    data=extract_advanced_features(df)\n    array_1=data.values[0]\n    array_2=df.describe().values.reshape(-1), filename.split('=')[1]\n    # Combine the two arrays\n    combined_array = np.concatenate((array_1, array_2[0]))\n    combined_tuple=(array_1,array_2[1])\n    return combined_tuple\n\ndef load_time_series(dirname) -> pd.DataFrame:\n    ids = os.listdir(dirname)\n\n    with ThreadPoolExecutor() as executor:\n        results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n\n    stats, indexes = zip(*results)\n\n    df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n    df['id'] = indexes\n    return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom sklearn.preprocessing import StandardScaler\nfrom torch.utils.data import DataLoader, TensorDataset\nfrom tqdm import tqdm\n\n\nclass Autoencoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(Autoencoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, input_dim),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return encoded, decoded\n\n\nclass DeepAutoencoder(nn.Module):\n    def __init__(self, input_dim, encoding_dim):\n        super(DeepAutoencoder, self).__init__()\n        self.encoder = nn.Sequential(\n            nn.Linear(input_dim, encoding_dim * 3),\n            nn.BatchNorm1d(encoding_dim * 3),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(encoding_dim * 3, encoding_dim * 2),\n            nn.BatchNorm1d(encoding_dim * 2),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(encoding_dim * 2, encoding_dim),\n            nn.BatchNorm1d(encoding_dim),\n            nn.ReLU()\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(encoding_dim, encoding_dim * 2),\n            nn.BatchNorm1d(encoding_dim * 2),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(encoding_dim * 2, encoding_dim * 3),\n            nn.BatchNorm1d(encoding_dim * 3),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(encoding_dim * 3, input_dim),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        encoded = self.encoder(x)\n        decoded = self.decoder(encoded)\n        return encoded, decoded\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32, learning_rate=0.001,\n                        use_deep=True, save_model_path=None, seed=42):\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    np.random.seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n    df.fillna(df.mean(), inplace=True)\n    df = df.loc[:, df.std() > 0]\n\n    scaler = StandardScaler()\n    df_scaled = scaler.fit_transform(df)\n\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    data_tensor = torch.tensor(df_scaled, dtype=torch.float32).to(device)\n\n    dataset = TensorDataset(data_tensor, data_tensor)\n    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)\n\n    input_dim = df_scaled.shape[1]\n    if use_deep:\n        model = DeepAutoencoder(input_dim=input_dim, encoding_dim=encoding_dim).to(device)\n    else:\n        model = Autoencoder(input_dim=input_dim, encoding_dim=encoding_dim).to(device)\n\n    criterion = nn.MSELoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5)\n\n    best_loss = float('inf')\n    losses = []\n    for epoch in range(epochs):\n        model.train()\n        epoch_loss = 0.0\n        progress_bar = tqdm(dataloader, desc=f\"Epoch {epoch + 1}/{epochs}\", leave=False)\n\n        for batch_features, _ in progress_bar:\n            batch_features = batch_features.to(device)\n\n            _, decoded = model(batch_features)\n            loss = criterion(decoded, batch_features)\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            epoch_loss += loss.item()\n            progress_bar.set_postfix(loss=loss.item())\n\n        epoch_loss /= len(dataloader)\n        losses.append(epoch_loss)\n\n        if epoch_loss < best_loss:\n            best_loss = epoch_loss\n            if save_model_path:\n                torch.save(model.state_dict(), save_model_path)\n                print(f\"Best model saved at Epoch {epoch + 1} with Loss: {best_loss:.4f}\")\n\n        scheduler.step(epoch_loss)\n        print(f\"Epoch {epoch + 1}/{epochs}, Avg Loss: {epoch_loss:.4f}\")\n\n    plt.plot(range(1, epochs + 1), losses)\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.title('Training Loss')\n    plt.show()\n\n    model.eval()\n    with torch.no_grad():\n        encoded_data = model.encoder(data_tensor).cpu().numpy()\n\n    df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n\n    return df_encoded\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def convert_object_to_category(df):\n    return df.astype({col: \"category\" for col in df.select_dtypes(include=[\"object\"]).columns})","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class OptunaRounder:\n    def __init__(self, y_true, y_pred):\n        self.y_true = np.array(y_true)\n        self.y_pred = np.array(y_pred)\n        self.labels = np.unique(self.y_true)\n\n    def __call__(self, trial):\n        thresholds = []\n        for i in range(len(self.labels) - 1):\n            low = thresholds[-1] if i > 0 else min(self.labels)\n            high = max(self.labels)\n            t = trial.suggest_uniform(f't{i}', low, high)\n            thresholds.append(t)\n\n        thresholds = sorted(thresholds)\n        try:\n            if not isinstance(self.y_true, (np.ndarray, list)):\n                raise ValueError(f\"Invalid y_true type: {type(self.y_true)}\")\n            if not isinstance(self.y_pred, np.ndarray):\n                raise ValueError(f\"Invalid y_pred type: {type(self.y_pred)}\")\n\n            opt_y_pred = self.adjust(self.y_pred, thresholds)\n            return cohen_kappa_score(self.y_true, opt_y_pred, weights='quadratic')\n        except Exception as e:\n            print(f\"Error in OptunaRounder call: {e}\")\n            return 0\n\n    def adjust(self, y_pred, thresholds):\n        if not isinstance(y_pred, np.ndarray):\n            y_pred = np.array(y_pred)\n        opt_y_pred = np.digitize(y_pred, [-np.inf] + thresholds + [np.inf]) - 1\n        return opt_y_pred\n\n    @staticmethod\n    def adjust_static(y_pred, thresholds):\n        return np.digitize(y_pred, [-np.inf] + thresholds + [np.inf]) - 1\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.read_csv(f\"{DATA_DIR}/train.csv\")\nts_train = load_time_series(f\"{DATA_DIR}/series_train.parquet\")\n# DeepAutoencoderを使用\ntrain_ts_encoded = perform_autoencoder(\n    df=ts_train.set_index(\"id\"),\n    encoding_dim=60,\n    epochs=100,\n    batch_size=32,\n    use_deep=True,               # DeepAutoencoderを使用\n    seed=SEED,\n    # save_model_path=\"deep_autoencoder_model.pth\"  # モデル保存パス\n)\n\n\ntrain_ts_encoded[\"id\"] = ts_train[\"id\"]\ntrain_ts_encoded.set_index(\"id\", inplace=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train, added_cols = feature_engineering(df_train)\nFEATURE_COLS = FEATURE_COLS + added_cols\n\ndf_train = pd.merge(df_train, train_ts_encoded, how=\"left\", on='id')\ndf_train = pd.merge(df_train, ts_train, how=\"left\", on='id')\ndf_train = df_train.set_index('id')\n\ntime_series_cols = ts_train.columns.tolist()\nts_encoded_cols = train_ts_encoded.columns.tolist()\ntime_series_cols.remove('id')\nFEATURE_COLS = FEATURE_COLS + ts_encoded_cols + time_series_cols\n\ndf_train = convert_object_to_category(df_train)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = df_train.dropna(subset=TARGET_COL)\ny = df_train[TARGET_COL]\nX = df_train[FEATURE_COLS]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"a = y.mean()\nb = y.var(ddof=0)\n\ny_min = y.min()\ny_max = y.max()\nprint(f\"y_min:{y_min}, y_max:{y_max}\")\n\ndef quadratic_weighted_kappa(preds, data):\n    y_true = data.get_label()\n    y_pred = preds.clip(y_min, y_max).round()\n    qwk = cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n    return 'qwk', qwk, True\n\ndef qwk_obj(preds, dtrain):\n    labels = dtrain.get_label()\n    preds = np.array(preds).clip(y_min, y_max)\n\n    f = 1/2 * np.sum((preds - labels) ** 2)\n    g = 1/2 * np.sum((preds - a) ** 2 + b)\n\n    df = preds - labels\n    dg = preds - a\n    grad = (df / g - f * dg / g ** 2) * len(labels)\n    hess = np.ones(len(labels))\n\n    return grad, hess\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optuna.logging.set_verbosity(optuna.logging.WARNING)\n\ndef objective_with_kfold(trial, X, y, n_splits=10, seed=42):\n    params = {\n        \"objective\": \"regression\",\n        \"metric\": \"None\",\n        \"verbosity\": -1,\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.005, 0.05, log=True),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 50),\n        \"min_data_in_leaf\": trial.suggest_int(\"min_data_in_leaf\", 10, 50),\n        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.5, 0.9),\n        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.5, 0.9),\n        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 10),\n        \"seed\": seed\n    }\n    params[\"objective\"] = qwk_obj\n\n    if gpu_available:\n        params.update({\n            \"device\": \"gpu\",\n            \"gpu_platform_id\": 0,\n            \"gpu_device_id\": 0\n        })\n\n    kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n    y_stratified = pd.qcut(y, q=n_splits, labels=False, duplicates=\"drop\")\n\n    fold_scores = []\n    fold_thresholds = []\n\n    for train_idx, valid_idx in kf.split(X, y_stratified):\n        X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n        y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n\n        lgb_train = lgb.Dataset(X_train, y_train)\n        lgb_valid = lgb.Dataset(X_valid, y_valid, reference=lgb_train)\n\n        model = lgb.train(\n            params=params,\n            train_set=lgb_train,\n            num_boost_round=10000,\n            valid_sets=[lgb_train, lgb_valid],\n            valid_names=[\"train\", \"valid\"],\n            feval=quadratic_weighted_kappa,\n            callbacks=[\n                lgb.early_stopping(stopping_rounds=100, verbose=True),\n                lgb.log_evaluation(100)\n            ]\n        )\n\n        y_pred = model.predict(X_valid, num_iteration=model.best_iteration)\n\n        rounder = OptunaRounder(y_valid, y_pred)\n        study = optuna.create_study(direction=\"maximize\")\n        study.optimize(rounder, n_trials=50)\n        best_thresholds = sorted(study.best_params.values())\n        fold_thresholds.append(best_thresholds)\n\n        y_pred_adjusted = rounder.adjust(y_pred, best_thresholds)\n        fold_score = cohen_kappa_score(y_valid, y_pred_adjusted, weights=\"quadratic\")\n        fold_scores.append(fold_score)\n\n    return np.mean(fold_scores), fold_thresholds\n\n\ndef optimize_with_kfold(X, y, n_trials=50, n_splits=10, seed=42):\n    trial_fold_thresholds = {}\n\n    def objective_wrapper(trial):\n        mean_score, fold_thresholds = objective_with_kfold(trial, X, y, n_splits, seed)\n        trial_fold_thresholds[trial.number] = fold_thresholds\n        return mean_score\n\n    study = optuna.create_study(direction=\"maximize\")\n    study.optimize(objective_wrapper, n_trials=n_trials)\n\n    best_trial_number = study.best_trial.number\n    best_fold_thresholds = trial_fold_thresholds[best_trial_number]\n\n    print(\"Best trial:\")\n    print(f\"  Value: {study.best_trial.value}\")\n    print(f\"  Params: {study.best_trial.params}\")\n    print(f\"Best Trial Number: {best_trial_number}\")\n    print(f\"Best Fold Thresholds: {best_fold_thresholds}\")\n\n    return study.best_trial.params, study, best_fold_thresholds\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"N_TRIALS = 50\nwandb.config.update({\"optimize_n_trials\": N_TRIALS})\nbest_params, study, fold_thresholds = optimize_with_kfold(X, y, n_trials=N_TRIALS, n_splits=10, seed=SEED)\n\nfile_path = os.path.join(save_path, f\"fold_thresholds_{SEED}.pkl\")\nwith open(file_path, \"wb\") as f:\n    pickle.dump(fold_thresholds, f)\n\noptuna_params = {\n    \"objective\": \"regression\",\n    \"metric\": \"None\",\n    \"verbosity\": -1,\n    \"learning_rate\": best_params[\"learning_rate\"],\n    \"num_leaves\": best_params[\"num_leaves\"],\n    \"min_data_in_leaf\": best_params[\"min_data_in_leaf\"],\n    \"feature_fraction\": best_params[\"feature_fraction\"],\n    \"bagging_fraction\": best_params[\"bagging_fraction\"],\n    \"bagging_freq\": best_params[\"bagging_freq\"],\n    \"seed\": SEED\n}\noptuna_params[\"objective\"] = qwk_obj\nif gpu_available:\n    optuna_params.update({\n        \"device\": \"gpu\",\n        \"gpu_platform_id\": 0,\n        \"gpu_device_id\": 0\n    })\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_values = [trial.value for trial in study.trials]\nplt.plot(best_values)\nplt.xlabel(\"Trial\")\nplt.ylabel(\"Best Value\")\nplt.title(\"Optuna Trials Performance\")\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds_oof = np.zeros(len(df_train))\n\nkf = StratifiedKFold(n_splits=10, shuffle=True, random_state=SEED)\ntry:\n    y_stratified = pd.qcut(y, q=kf.get_n_splits(), labels=False, duplicates=\"drop\")\nexcept ValueError:\n    y_stratified = y\n\nfold_scores = []\nmodels = []\n\nfor fold, (idx_train, idx_valid) in enumerate(kf.split(X, y_stratified)):\n    X_train = X.iloc[idx_train]\n    y_train = y.iloc[idx_train]\n    X_valid = X.iloc[idx_valid]\n    y_valid = y.iloc[idx_valid]\n\n    lgb_train = lgb.Dataset(X_train, y_train)\n    lgb_valid = lgb.Dataset(X_valid, y_valid)\n\n    callbacks = [\n        lgb.early_stopping(stopping_rounds=100, verbose=True),\n        lgb.log_evaluation(100)\n    ]\n\n    model = lgb.train(\n        params=optuna_params,\n        train_set=lgb_train,\n        num_boost_round=10000,\n        valid_sets=[lgb_train, lgb_valid],\n        valid_names=[\"train\", \"valid\"],\n        feval=quadratic_weighted_kappa,\n        callbacks=callbacks\n    )\n\n    y_pred = model.predict(X_valid, num_iteration=model.best_iteration)\n    best_thresholds = fold_thresholds[fold]\n    y_pred_adjusted = OptunaRounder.adjust_static(y_pred, best_thresholds)\n    fold_score = cohen_kappa_score(y_valid, y_pred_adjusted, weights=\"quadratic\")\n    fold_scores.append(fold_score)\n    wandb.log({f\"fold_{fold + 1}_qwk_score\": fold_score})\n\n    models.append(model)\n    preds_oof[idx_valid] = y_pred_adjusted\n\ncv_score = cohen_kappa_score(y, preds_oof, weights=\"quadratic\")\n\naverage_fold_score = np.mean(fold_scores)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}