{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":210.897109,"end_time":"2024-11-20T15:19:03.850234","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-11-20T15:15:32.953125","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\nimport numpy as np\n\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report,r2_score\n\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom sklearn.model_selection import train_test_split\n\nfrom xgboost import XGBClassifier\n\nfrom catboost import CatBoostClassifier, CatBoostRegressor\n\nfrom sklearn.metrics import cohen_kappa_score\n\nimport optuna\n\nfrom catboost import Pool\n\n\n\nimport pandas as pd\n\nimport os\n\nfrom tqdm import tqdm\n\nfrom concurrent.futures import ThreadPoolExecutor\n\n\n\nimport pandas as pd\n\nimport numpy as np\n\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report\n\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom sklearn.model_selection import train_test_split\n\nfrom xgboost import XGBClassifier\n\nfrom catboost import CatBoostClassifier, CatBoostRegressor\n\nfrom sklearn.metrics import cohen_kappa_score\n\nimport optuna\n\nfrom catboost import Pool\n\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm import tqdm\n\nfrom IPython.display import clear_output\n\nfrom scipy.optimize import minimize\n\nfrom colorama import Fore, Style # 确保导入 Fore 和 Style\n\nimport torch\n\nimport torch.nn as nn\n\nimport torch.optim as optim\n\nfrom sklearn.preprocessing import StandardScaler\n\nfrom concurrent.futures import ThreadPoolExecutor\n\nimport os\n\nimport polars as pl\n\nimport matplotlib.pyplot as plt\n\nfrom matplotlib.ticker import PercentFormatter\n\nimport seaborn as sns\n\nimport random\n\n\n\nSEED = 42 \n\nnp.random.seed(SEED)\n\ntorch.manual_seed(SEED)\n\nrandom.seed(SEED)\n\nif torch.cuda.is_available():\n\n torch.cuda.manual_seed(SEED)\n\n torch.cuda.manual_seed_all(SEED) # 如果有多个GPU\n\n torch.backends.cudnn.deterministic = True\n\n torch.backends.cudnn.benchmark = False","metadata":{"papermill":{"duration":6.218633,"end_time":"2024-11-20T15:15:41.657666","exception":false,"start_time":"2024-11-20T15:15:35.439033","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:53:17.775416Z","iopub.execute_input":"2024-12-12T14:53:17.775780Z","iopub.status.idle":"2024-12-12T14:53:17.788530Z","shell.execute_reply.started":"2024-12-12T14:53:17.775751Z","shell.execute_reply":"2024-12-12T14:53:17.787759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_file(filename, dirname):\n\n df = pd.read_parquet(os.path.join(dirname, filename, 'part-0.parquet'))\n\n df.drop('step', axis=1, inplace=True)\n\n return df.describe().values.reshape(-1), filename.split('=')[1]\n\n\n\ndef load_time_series(dirname) -> pd.DataFrame:\n\n ids = os.listdir(dirname)\n\n \n\n with ThreadPoolExecutor() as executor:\n\n results = list(tqdm(executor.map(lambda fname: process_file(fname, dirname), ids), total=len(ids)))\n\n \n\n stats, indexes = zip(*results)\n\n \n\n df = pd.DataFrame(stats, columns=[f\"stat_{i}\" for i in range(len(stats[0]))])\n\n df['id'] = indexes\n\n return df","metadata":{"papermill":{"duration":0.022189,"end_time":"2024-11-20T15:15:41.693931","exception":false,"start_time":"2024-11-20T15:15:41.671742","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:53:17.789997Z","iopub.execute_input":"2024-12-12T14:53:17.790296Z","iopub.status.idle":"2024-12-12T14:53:17.801851Z","shell.execute_reply.started":"2024-12-12T14:53:17.790270Z","shell.execute_reply":"2024-12-12T14:53:17.800962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class AutoEncoder(nn.Module):\n\n def __init__(self, input_dim, encoding_dim):\n\n super(AutoEncoder, self).__init__()\n\n self.encoder = nn.Sequential(\n\n nn.Linear(input_dim, encoding_dim*3),\n\n nn.ReLU(),\n\n nn.Linear(encoding_dim*3, encoding_dim*2),\n\n nn.ReLU(),\n\n nn.Linear(encoding_dim*2, encoding_dim),\n\n nn.ReLU()\n\n )\n\n self.decoder = nn.Sequential(\n\n nn.Linear(encoding_dim, input_dim*2),\n\n nn.ReLU(),\n\n nn.Linear(input_dim*2, input_dim*3),\n\n nn.ReLU(),\n\n nn.Linear(input_dim*3, input_dim),\n\n nn.Sigmoid()\n\n )\n\n \n\n def forward(self, x):\n\n encoded = self.encoder(x)\n\n decoded = self.decoder(encoded)\n\n return decoded\n\n\n\n\n\ndef perform_autoencoder(df, encoding_dim=50, epochs=50, batch_size=32):\n\n scaler = StandardScaler()\n\n df_scaled = scaler.fit_transform(df)\n\n \n\n data_tensor = torch.FloatTensor(df_scaled)\n\n \n\n input_dim = data_tensor.shape[1]\n\n autoencoder = AutoEncoder(input_dim, encoding_dim)\n\n \n\n criterion = nn.MSELoss()\n\n optimizer = optim.Adam(autoencoder.parameters())\n\n \n\n for epoch in range(epochs):\n\n for i in range(0, len(data_tensor), batch_size):\n\n batch = data_tensor[i : i + batch_size]\n\n optimizer.zero_grad()\n\n reconstructed = autoencoder(batch)\n\n loss = criterion(reconstructed, batch)\n\n loss.backward()\n\n optimizer.step()\n\n \n\n if (epoch + 1) % 10 == 0:\n\n print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss.item():.4f}]')\n\n \n\n with torch.no_grad():\n\n encoded_data = autoencoder.encoder(data_tensor).numpy()\n\n \n\n df_encoded = pd.DataFrame(encoded_data, columns=[f'Enc_{i + 1}' for i in range(encoded_data.shape[1])])\n\n \n\n return df_encoded\n\n\n\ndef feature_engineering(df):\n\n season_cols = [col for col in df.columns if 'Season' in col]\n\n df = df.drop(season_cols, axis=1) \n\n df['BMI_Age'] = df['Physical-BMI'] * df['Basic_Demos-Age']\n\n df['Internet_Hours_Age'] = df['PreInt_EduHx-computerinternet_hoursday'] * df['Basic_Demos-Age']\n\n df['BMI_Internet_Hours'] = df['Physical-BMI'] * df['PreInt_EduHx-computerinternet_hoursday']\n\n df['BFP_BMI'] = df['BIA-BIA_Fat'] / df['BIA-BIA_BMI']\n\n df['FFMI_BFP'] = df['BIA-BIA_FFMI'] / df['BIA-BIA_Fat']\n\n df['FMI_BFP'] = df['BIA-BIA_FMI'] / df['BIA-BIA_Fat']\n\n df['LST_TBW'] = df['BIA-BIA_LST'] / df['BIA-BIA_TBW']\n\n df['BFP_BMR'] = df['BIA-BIA_Fat'] * df['BIA-BIA_BMR']\n\n df['BFP_DEE'] = df['BIA-BIA_Fat'] * df['BIA-BIA_DEE']\n\n df['BMR_Weight'] = df['BIA-BIA_BMR'] / df['Physical-Weight']\n\n df['DEE_Weight'] = df['BIA-BIA_DEE'] / df['Physical-Weight']\n\n df['SMM_Height'] = df['BIA-BIA_SMM'] / df['Physical-Height']\n\n df['Muscle_to_Fat'] = df['BIA-BIA_SMM'] / df['BIA-BIA_FMI']\n\n # df['Hydration_Status'] = df['BIA-BIA_TBW'] / df['Physical-Weight']\n\n # df['ICW_TBW'] = df['BIA-BIA_ICW'] / df['BIA-BIA_TBW']\n\n \n\n return df","metadata":{"papermill":{"duration":0.028772,"end_time":"2024-11-20T15:15:41.735903","exception":false,"start_time":"2024-11-20T15:15:41.707131","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:53:17.803120Z","iopub.execute_input":"2024-12-12T14:53:17.803402Z","iopub.status.idle":"2024-12-12T14:53:17.820397Z","shell.execute_reply.started":"2024-12-12T14:53:17.803377Z","shell.execute_reply":"2024-12-12T14:53:17.819515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\n\ntest = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n\nsample = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv')\n\n\n\ntrain_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet\")\n\ntest_ts = load_time_series(\"/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet\")\n\n\n\nprint(f\"train shape: {train.shape}\")\n\nprint(f\"test shape: {test.shape}\")\n\nprint(f\"train_ts shape: {train_ts.shape}\")\n\nprint(f\"test_ts shape: {test_ts.shape}\")","metadata":{"papermill":{"duration":73.516249,"end_time":"2024-11-20T15:16:55.265228","exception":false,"start_time":"2024-11-20T15:15:41.748979","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:53:17.902221Z","iopub.execute_input":"2024-12-12T14:53:17.903172Z","iopub.status.idle":"2024-12-12T14:54:32.506769Z","shell.execute_reply.started":"2024-12-12T14:53:17.903135Z","shell.execute_reply":"2024-12-12T14:54:32.505890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 获取 test 数据中的列名\n\ntest_columns = test.columns\n\ntest_columns = list(test.columns)\n\ntest_columns.append('sii')\n\n\n\n# 在 train 中只保留 test 中的列\n\ntrain = train[test_columns]\n\n\n\n# 删除 train 中 sii 列值为空的行\n\ntrain = train.dropna(subset=['sii'])\n\n\n\nprint(f\"train shape: {train.shape}\")","metadata":{"papermill":{"duration":0.042151,"end_time":"2024-11-20T15:16:55.334738","exception":false,"start_time":"2024-11-20T15:16:55.292587","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:32.508621Z","iopub.execute_input":"2024-12-12T14:54:32.508890Z","iopub.status.idle":"2024-12-12T14:54:32.518541Z","shell.execute_reply.started":"2024-12-12T14:54:32.508863Z","shell.execute_reply":"2024-12-12T14:54:32.517551Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"进行标签编码","metadata":{"papermill":{"duration":0.026701,"end_time":"2024-11-20T15:16:55.388590","exception":false,"start_time":"2024-11-20T15:16:55.361889","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 更新数值型和类别型列\n\nnumerical_columns = train.select_dtypes(include=['int64', 'float64']).columns.tolist()\n\ncategorical_columns = train.select_dtypes(include=['object']).columns.tolist()\n\n\n\nprint(numerical_columns)\n\n# 对 train 和 test 中的数据类型为类别的列进行标签编码\n\ncolumns_to_encode_df = categorical_columns\n\ncolumns_to_encode_test = test.select_dtypes(include=['object']).columns.tolist()\n\n\n\n# 排除 'id' 列，防止对其进行编码\n\ncolumns_to_encode_test = [col for col in columns_to_encode_test if col != 'id']\n\ncolumns_to_encode_df = [col for col in columns_to_encode_df if col != 'id']\n\nlabel_encoder = LabelEncoder()\n\n\n\n# 对 train 中的类别列进行标签编码\n\nfor column in columns_to_encode_df:\n\n train[column] = label_encoder.fit_transform(train[column])\n\n\n\n# 对 test 中的类别列进行标签编码\n\nfor column in columns_to_encode_test:\n\n test[column] = label_encoder.fit_transform(test[column])\n\n\n\nprint(f\"train shape: {train.shape}\")\n\nprint(f\"test shape: {test.shape}\")\n\nprint(f\"train_ts shape: {train_ts.shape}\")\n\nprint(f\"test_ts shape: {test_ts.shape}\")\n\n\n\ndata_types_train = train.dtypes\n\nprint(\"每一列的数据类型：\")\n\nprint(data_types_train)\n\n\n\ndata_types_test = test.dtypes\n\nprint(\"每一列的数据类型：\")\n\nprint(data_types_test)","metadata":{"papermill":{"duration":0.055813,"end_time":"2024-11-20T15:16:55.471732","exception":false,"start_time":"2024-11-20T15:16:55.415919","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:32.519640Z","iopub.execute_input":"2024-12-12T14:54:32.520005Z","iopub.status.idle":"2024-12-12T14:54:32.547589Z","shell.execute_reply.started":"2024-12-12T14:54:32.519953Z","shell.execute_reply":"2024-12-12T14:54:32.546642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\ntrain_with_time = pd.merge(train, train_ts, on='id', how='inner')\n\n\n\n# 从 train 中删除已经合并到 train_with_time 中的行，形成新的 train\n\ntrain = train[~train['id'].isin(train_with_time['id'])]\n\ntrain_with_time = train_with_time.drop(columns=['id'])\n\ntrain = train.drop(columns=['id'])\n\n\n\n# 检查结果\n\nprint(\"train_with_time shape:\", train_with_time.shape)\n\nprint(\"train shape:\", train.shape)\n\n\n\n\n\ntest_with_time = pd.merge(test, test_ts, on='id', how='inner')\n\n\n\n# 从 train 中删除已经合并到 train_with_time 中的行，形成新的 train\n\n# test = test[~test['id'].isin(test_with_time['id'])]\n\n\n\n# 检查结果\n\nprint(\"test_with_time shape:\", test_with_time.shape)\n\nprint(\"test shape:\", test.shape)","metadata":{"papermill":{"duration":0.062473,"end_time":"2024-11-20T15:16:55.561779","exception":false,"start_time":"2024-11-20T15:16:55.499306","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:32.548911Z","iopub.execute_input":"2024-12-12T14:54:32.549231Z","iopub.status.idle":"2024-12-12T14:54:32.567436Z","shell.execute_reply.started":"2024-12-12T14:54:32.549202Z","shell.execute_reply":"2024-12-12T14:54:32.566594Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"train 中的缺失值数目：\")\n\nprint(train.isnull().sum())\n\nprint(\"test 中的缺失值数目：\")\n\nprint(test.isnull().sum())","metadata":{"papermill":{"duration":0.040767,"end_time":"2024-11-20T15:16:55.631192","exception":false,"start_time":"2024-11-20T15:16:55.590425","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:32.569604Z","iopub.execute_input":"2024-12-12T14:54:32.569883Z","iopub.status.idle":"2024-12-12T14:54:32.579266Z","shell.execute_reply.started":"2024-12-12T14:54:32.569857Z","shell.execute_reply":"2024-12-12T14:54:32.578324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 过滤掉 'sii' 列中非空的行\n\nsupervised_usable = train[train['sii'].notnull()]\n\n\n\n# 统计缺失值数量和比例\n\nmissing_count = supervised_usable.isnull().sum().reset_index()\n\nmissing_count.columns = ['feature', 'null_count']\n\nmissing_count['null_ratio'] = missing_count['null_count'] / len(supervised_usable)\n\n\n\n# 绘制缺失值图表\n\nplt.figure(figsize=(6, 15))\n\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\n\nplt.barh(np.arange(len(missing_count)), missing_count['null_ratio'], color='coral', label='missing')\n\nplt.barh(np.arange(len(missing_count)), 1 - missing_count['null_ratio'], \n\n left=missing_count['null_ratio'], color='darkseagreen', label='available')\n\nplt.yticks(np.arange(len(missing_count)), missing_count['feature'])\n\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\n\nplt.xlim(0, 1)\n\nplt.legend()\n\nplt.show()","metadata":{"papermill":{"duration":0.814801,"end_time":"2024-11-20T15:16:56.475197","exception":false,"start_time":"2024-11-20T15:16:55.660396","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:32.580469Z","iopub.execute_input":"2024-12-12T14:54:32.580760Z","iopub.status.idle":"2024-12-12T14:54:33.305312Z","shell.execute_reply.started":"2024-12-12T14:54:32.580732Z","shell.execute_reply":"2024-12-12T14:54:33.304379Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 过滤掉 'sii' 列中非空的行\n\nsupervised_usable = train_with_time[train_with_time['sii'].notnull()]\n\n\n\n# 统计缺失值数量和比例\n\nmissing_count = supervised_usable.isnull().sum().reset_index()\n\nmissing_count.columns = ['feature', 'null_count']\n\nmissing_count['null_ratio'] = missing_count['null_count'] / len(supervised_usable)\n\n\n\n# 绘制缺失值图表\n\nplt.figure(figsize=(6, 30))\n\nplt.title(f'Missing values over the {len(supervised_usable)} samples which have a target')\n\nplt.barh(np.arange(len(missing_count)), missing_count['null_ratio'], color='coral', label='missing')\n\nplt.barh(np.arange(len(missing_count)), 1 - missing_count['null_ratio'], \n\n left=missing_count['null_ratio'], color='darkseagreen', label='available')\n\nplt.yticks(np.arange(len(missing_count)), missing_count['feature'])\n\nplt.gca().xaxis.set_major_formatter(PercentFormatter(xmax=1, decimals=0))\n\nplt.xlim(0, 1)\n\nplt.legend()\n\nplt.show()\n","metadata":{"papermill":{"duration":1.458087,"end_time":"2024-11-20T15:16:57.963747","exception":false,"start_time":"2024-11-20T15:16:56.505660","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:33.306681Z","iopub.execute_input":"2024-12-12T14:54:33.307024Z","iopub.status.idle":"2024-12-12T14:54:34.946233Z","shell.execute_reply.started":"2024-12-12T14:54:33.306990Z","shell.execute_reply":"2024-12-12T14:54:34.945239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 计算相关矩阵\n\ncorr_matrix = train.corr()\n\n\n\n# 设置绘图大小\n\nplt.figure(figsize=(24, 20))\n\n\n\n# 绘制热力图\n\nsns.heatmap(corr_matrix, annot=False, cmap='coolwarm', center=0, \n\n square=True, linewidths=.5, cbar_kws={\"shrink\": .8})\n\n\n\nplt.title(\"Correlation Heatmap of Train Dataset\")\n\nplt.show()\n","metadata":{"papermill":{"duration":1.200482,"end_time":"2024-11-20T15:16:59.199692","exception":false,"start_time":"2024-11-20T15:16:57.999210","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:34.947619Z","iopub.execute_input":"2024-12-12T14:54:34.947903Z","iopub.status.idle":"2024-12-12T14:54:36.160147Z","shell.execute_reply.started":"2024-12-12T14:54:34.947875Z","shell.execute_reply":"2024-12-12T14:54:36.159240Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"合并一下数据集，增强数据量","metadata":{"papermill":{"duration":0.039121,"end_time":"2024-11-20T15:16:59.279287","exception":false,"start_time":"2024-11-20T15:16:59.240166","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 1. 按行合并两个 DataFrame\n\ncombined_df = pd.concat([train, train_with_time], ignore_index=True)\n\n\n\n# 2. 选择 train 中已有的列\n\ncombined_df = combined_df[train.columns]\n\ntrain = combined_df\n\n# 打印合并后的 DataFrame 以验证\n\nprint(\"合并后的 DataFrame:\")\n\nprint(combined_df.head())\n\nprint(\"DataFrame 的形状：\", combined_df.shape)","metadata":{"papermill":{"duration":0.060978,"end_time":"2024-11-20T15:16:59.378140","exception":false,"start_time":"2024-11-20T15:16:59.317162","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:36.161268Z","iopub.execute_input":"2024-12-12T14:54:36.161524Z","iopub.status.idle":"2024-12-12T14:54:36.182160Z","shell.execute_reply.started":"2024-12-12T14:54:36.161499Z","shell.execute_reply":"2024-12-12T14:54:36.181242Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"进行数据分割","metadata":{"papermill":{"duration":0.037879,"end_time":"2024-11-20T15:16:59.454829","exception":false,"start_time":"2024-11-20T15:16:59.416950","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_data, valid_data = train_test_split(\n\n train,\n\n test_size=0.2,\n\n stratify=train['sii'],\n\n random_state=42 # 设置随机种子保证结果可重复\n\n)\n\n\n\n# 检查 'sii' 列中类别比例是否一致\n\nprint(\"Train set 'sii' 类别比例:\")\n\nprint(train_data['sii'].value_counts(normalize=True))\n\nprint(\"\\nValid set 'sii' 类别比例:\")\n\nprint(valid_data['sii'].value_counts(normalize=True))\n\n\n\ntrain_data_withtime, valid_data_withtime = train_test_split(\n\n train_with_time,\n\n test_size=0.2,\n\n stratify=train_with_time['sii'],\n\n random_state=42 # 设置随机种子保证结果可重复\n\n)\n\n\n\n# 检查 'sii' 列中类别比例是否一致\n\nprint(\"Train set 'sii' 类别比例:\")\n\nprint(train_data_withtime['sii'].value_counts(normalize=True))\n\nprint(\"\\nValid set 'sii' 类别比例:\")\n\nprint(valid_data_withtime['sii'].value_counts(normalize=True))","metadata":{"papermill":{"duration":0.061964,"end_time":"2024-11-20T15:16:59.555054","exception":false,"start_time":"2024-11-20T15:16:59.493090","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:36.183824Z","iopub.execute_input":"2024-12-12T14:54:36.184265Z","iopub.status.idle":"2024-12-12T14:54:36.205451Z","shell.execute_reply.started":"2024-12-12T14:54:36.184219Z","shell.execute_reply":"2024-12-12T14:54:36.204262Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"确保valid和test一样","metadata":{"papermill":{"duration":0.039831,"end_time":"2024-11-20T15:16:59.680709","exception":false,"start_time":"2024-11-20T15:16:59.640878","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 提取 valid 数据集中的 sii 列并保存到单独的变量中\n\nvalid_sii = valid_data['sii'].copy()\n\n\n\n# 删除 valid 数据集中的 sii 列\n\nvalid_data = valid_data.drop(columns=['sii'])\n\n\n\n# 提取 valid 数据集中的 sii 列并保存到单独的变量中\n\nvalid_sii_withtime = valid_data_withtime['sii'].copy()\n\n\n\n# 删除 valid 数据集中的 sii 列\n\nvalid_data_withtime = valid_data_withtime.drop(columns=['sii'])","metadata":{"papermill":{"duration":0.048169,"end_time":"2024-11-20T15:16:59.767953","exception":false,"start_time":"2024-11-20T15:16:59.719784","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:36.206833Z","iopub.execute_input":"2024-12-12T14:54:36.207354Z","iopub.status.idle":"2024-12-12T14:54:36.215007Z","shell.execute_reply.started":"2024-12-12T14:54:36.207311Z","shell.execute_reply":"2024-12-12T14:54:36.214109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 数据预处理函数\n\ndef standardize_data(data):\n\n col_means = np.nanmean(data, axis=0)\n\n col_stds = np.nanstd(data, axis=0)\n\n standardized_data = (data - col_means) / col_stds\n\n return standardized_data, col_means, col_stds\n\n\n\ndef inverse_transform(standardized_data, col_means, col_stds):\n\n data = standardized_data * col_stds + col_means\n\n return data\n\n\n","metadata":{"papermill":{"duration":0.04658,"end_time":"2024-11-20T15:16:59.852834","exception":false,"start_time":"2024-11-20T15:16:59.806254","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:36.216205Z","iopub.execute_input":"2024-12-12T14:54:36.216504Z","iopub.status.idle":"2024-12-12T14:54:36.223688Z","shell.execute_reply.started":"2024-12-12T14:54:36.216479Z","shell.execute_reply":"2024-12-12T14:54:36.222748Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"BASELINE","metadata":{"papermill":{"duration":0.037649,"end_time":"2024-11-20T15:16:59.928507","exception":false,"start_time":"2024-11-20T15:16:59.890858","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from sklearn.impute import SimpleImputer\n\n# import pandas as pd\n\n\n\n# # 1. 提取 'sii' 列\n\n# sii_column = train_data['sii'].values\n\n# train_features = train_data.drop(columns=['sii'])\n\n\n\n# # 2. 合并 train_features 和 valid_data\n\n# combined_data = pd.concat([train_features, valid_data], ignore_index=True)\n\n\n\n# # 3. 识别数值型和非数值型列\n\n# numeric_cols = combined_data.select_dtypes(include=['number']).columns\n\n\n\n# # 4. 对数值型列进行缺失值填充（中位数）\n\n# numeric_imputer = SimpleImputer(strategy='median')\n\n# combined_data[numeric_cols] = numeric_imputer.fit_transform(combined_data[numeric_cols])\n\n\n\n\n\n# # 6. 将数据拆分回 train_data 和 valid_data\n\n# train_filled_df = combined_data.iloc[:len(train_features)].reset_index(drop=True)\n\n# valid_filled_df = combined_data.iloc[len(train_features):].reset_index(drop=True)\n\n\n\n# # 7. 将 'sii' 列添加回 train_filled_df\n\n# train_filled_df['sii'] = sii_column\n\n\n\n# # 8. 更新 train_data 和 valid_data\n\n# train_data = train_filled_df\n\n# valid_data = valid_filled_df\n\n\n\n# print(\"train_data 和 valid_data 缺失值填补完毕\")\n","metadata":{"papermill":{"duration":0.045475,"end_time":"2024-11-20T15:17:00.011985","exception":false,"start_time":"2024-11-20T15:16:59.966510","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:36.225021Z","iopub.execute_input":"2024-12-12T14:54:36.225302Z","iopub.status.idle":"2024-12-12T14:54:36.234490Z","shell.execute_reply.started":"2024-12-12T14:54:36.225278Z","shell.execute_reply":"2024-12-12T14:54:36.233647Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"缺失值填充方法1：创建并训练VAEAC","metadata":{"papermill":{"duration":0.03894,"end_time":"2024-11-20T15:17:00.089555","exception":false,"start_time":"2024-11-20T15:17:00.050615","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 定义Encoder、Decoder和VAEAC模型\n\nclass Encoder(nn.Module):\n\n def __init__(self, input_dim, latent_dim):\n\n super(Encoder, self).__init__()\n\n self.fc1 = nn.Linear(input_dim * 2, 128)\n\n self.fc2 = nn.Linear(128, 64)\n\n self.fc3_mean = nn.Linear(64, latent_dim)\n\n self.fc3_logvar = nn.Linear(64, latent_dim)\n\n \n\n def forward(self, x, mask):\n\n h = torch.relu(self.fc1(torch.cat([x, mask], dim=1)))\n\n h = torch.relu(self.fc2(h))\n\n mean = self.fc3_mean(h)\n\n logvar = self.fc3_logvar(h)\n\n return mean, logvar\n\n\n\nclass Decoder(nn.Module):\n\n def __init__(self, latent_dim, output_dim):\n\n super(Decoder, self).__init__()\n\n self.fc1 = nn.Linear(latent_dim + output_dim, 64)\n\n self.fc2 = nn.Linear(64, 128)\n\n self.fc3 = nn.Linear(128, output_dim)\n\n \n\n def forward(self, z, mask):\n\n h = torch.relu(self.fc1(torch.cat([z, mask], dim=1)))\n\n h = torch.relu(self.fc2(h))\n\n output = self.fc3(h)\n\n return output\n\n\n\nclass VAEAC(nn.Module):\n\n def __init__(self, input_dim, latent_dim):\n\n super(VAEAC, self).__init__()\n\n self.encoder = Encoder(input_dim, latent_dim)\n\n self.decoder = Decoder(latent_dim, input_dim)\n\n \n\n def reparameterize(self, mean, logvar):\n\n std = torch.exp(0.5 * logvar)\n\n eps = torch.randn_like(std)\n\n return mean + eps * std\n\n \n\n def forward(self, x, mask):\n\n mean, logvar = self.encoder(x, mask)\n\n z = self.reparameterize(mean, logvar)\n\n reconstructed_x = self.decoder(z, mask)\n\n return reconstructed_x, mean, logvar\n\n\n\n# 定义VAEAC的训练过程\n\ndef train_vaeac(model, data, mask, num_epochs=100, learning_rate=0.001):\n\n optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n\n criterion = nn.MSELoss(reduction=\"sum\")\n\n for epoch in range(num_epochs):\n\n model.train()\n\n optimizer.zero_grad()\n\n \n\n data_tensor = torch.FloatTensor(data)\n\n mask_tensor = torch.FloatTensor(mask)\n\n reconstructed, mean, logvar = model(data_tensor, mask_tensor)\n\n \n\n # 仅在观测值上计算重建损失\n\n recon_loss = criterion(reconstructed * mask_tensor, data_tensor * mask_tensor) / mask_tensor.sum()\n\n \n\n # KL散度损失\n\n kl_loss = -0.5 * torch.sum(1 + logvar - mean.pow(2) - logvar.exp()) / data.shape[0]\n\n \n\n loss = recon_loss + kl_loss\n\n loss.backward()\n\n optimizer.step()\n\n \n\n if (epoch + 1) % 10 == 0:\n\n print(f\"Epoch [{epoch + 1}/{num_epochs}], Loss: {loss.item():.4f}\")\n\n ","metadata":{"papermill":{"duration":0.056093,"end_time":"2024-11-20T15:17:00.184958","exception":false,"start_time":"2024-11-20T15:17:00.128865","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:36.239378Z","iopub.execute_input":"2024-12-12T14:54:36.239651Z","iopub.status.idle":"2024-12-12T14:54:36.255206Z","shell.execute_reply.started":"2024-12-12T14:54:36.239625Z","shell.execute_reply":"2024-12-12T14:54:36.254332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = feature_engineering(train_data)\n\nvalid_data = feature_engineering(valid_data)\ntest = feature_engineering(test)\n\n\n\nprint(f\"train shape: {train_data.shape}\")\n\nprint(\"train 中的缺失值数目：\")\n\nprint(train_data.isnull().sum())\n\n\n\n\n\nprint(f\"valid shape: {valid_data.shape}\")\n\nprint(\"valid 中的缺失值数目：\")\n\nprint(valid_data.isnull().sum())\n\n\n\n\n\ntrain_data_withtime = feature_engineering(train_data_withtime)\n\nvalid_data_withtime = feature_engineering(valid_data_withtime)\n\n\n\nprint(f\"train_withtime shape: {train_data_withtime.shape}\")\n\nprint(\"train_withtime 中的缺失值数目：\")\n\nprint(train_data_withtime.isnull().sum())\n\n\n\n\n\nprint(f\"valid shape_withtime: {valid_data_withtime.shape}\")\n\nprint(\"valid_withtime 中的缺失值数目：\")\n\nprint(valid_data_withtime.isnull().sum())\n\n\n\ntest = feature_engineering(test)\n\nprint(\"test 中的缺失值数目：\")\n\nprint(test.isnull().sum())","metadata":{"papermill":{"duration":0.091525,"end_time":"2024-11-20T15:17:00.315240","exception":false,"start_time":"2024-11-20T15:17:00.223715","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:36.256437Z","iopub.execute_input":"2024-12-12T14:54:36.256701Z","iopub.status.idle":"2024-12-12T14:54:36.313876Z","shell.execute_reply.started":"2024-12-12T14:54:36.256677Z","shell.execute_reply":"2024-12-12T14:54:36.312939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # 将 'sii' 列从 train_data 中分离，保持不变\n\n# sii_column = train_data['sii'].values\n\n# train_features = train_data.drop(columns=['sii'])\n\n# valid_features = valid_data.copy()\n\n\n\n# # 合并 train_features 和 valid_features\n\n# combined_features = pd.concat([train_features, valid_features], ignore_index=True)\n\n\n\n# # 转换为 numpy 数组\n\n# combined_values = combined_features.values\n\n\n\n# # 数据标准化，保留 NaN\n\n# standardized_data, col_means, col_stds = standardize_data(combined_values)\n\n# nan_mask = np.isnan(standardized_data)\n\n# observed_mask = ~nan_mask\n\n\n\n# # 将 NaN 替换为 0，仅用于模型输入\n\n# data_filled = np.nan_to_num(standardized_data, nan=0.0)\n\n\n\n# # 创建并训练 VAEAC 模型\n\n# input_dim = data_filled.shape[1]\n\n# latent_dim = 50 # 可以根据需要调整\n\n# vaeac_model = VAEAC(input_dim=input_dim, latent_dim=latent_dim)\n\n\n\n# train_vaeac(vaeac_model, data_filled, observed_mask, num_epochs=100, learning_rate=0.001)\n\n\n\n# # 填补缺失值\n\n# vaeac_model.eval()\n\n# with torch.no_grad():\n\n# data_tensor = torch.FloatTensor(data_filled)\n\n# mask_tensor = torch.FloatTensor(observed_mask)\n\n \n\n# mask_tensor_bool = mask_tensor.bool()\n\n \n\n# reconstructed, _, _ = vaeac_model(data_tensor, mask_tensor)\n\n \n\n# # 用重建值填补缺失值\n\n# filled_data = data_tensor * mask_tensor_bool + reconstructed * (~mask_tensor_bool)\n\n# filled_data = filled_data.numpy()\n\n\n\n# # 逆标准化回原始尺度\n\n# filled_data = inverse_transform(filled_data, col_means, col_stds)\n\n\n\n# # 将填充后的数据拆分回 train 和 valid\n\n# train_filled_data = filled_data[:len(train_features)]\n\n# valid_filled_data = filled_data[len(train_features):]\n\n\n\n# # 创建 DataFrame\n\n# train_filled_df = pd.DataFrame(train_filled_data, columns=train_features.columns)\n\n# valid_filled_df = pd.DataFrame(valid_filled_data, columns=valid_features.columns)\n\n\n\n# # 将 'sii' 列重新加入 train_filled_df\n\n# train_filled_df['sii'] = sii_column\n\n\n\n# # 现在，train_filled_df 和 valid_filled_df 就是填充了缺失值的数据集\n\n# # 您可以继续进行后续的处理\n\n\n\n# # 可选地，保存到 CSV 文件\n\n# train_filled_df.to_csv(\"train_filled.csv\", index=False)\n\n# valid_filled_df.to_csv(\"valid_filled.csv\", index=False)\n\n\n\n# train_data = train_filled_df\n\n# valid_data = valid_filled_df\n\n\n\n# print(\"train_data 和 valid_data 缺失值填补完毕\")","metadata":{"papermill":{"duration":0.056986,"end_time":"2024-11-20T15:17:00.412169","exception":false,"start_time":"2024-11-20T15:17:00.355183","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:36.315226Z","iopub.execute_input":"2024-12-12T14:54:36.315528Z","iopub.status.idle":"2024-12-12T14:54:36.321039Z","shell.execute_reply.started":"2024-12-12T14:54:36.315499Z","shell.execute_reply":"2024-12-12T14:54:36.320183Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"填充方法2：使用KNN填充（kaggle方法）","metadata":{"papermill":{"duration":0.039253,"end_time":"2024-11-20T15:17:00.491540","exception":false,"start_time":"2024-11-20T15:17:00.452287","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer, KNNImputer\n\nfrom sklearn.experimental import enable_iterative_imputer\n\nfrom sklearn.impute import IterativeImputer\n\nimport pandas as pd\n\n\n\n# 初始化 imputer\n\nimputer = KNNImputer(n_neighbors=5)\n\n\n\n# 1. 提取 'sii' 列\n\nsii_column = train_data['sii'].values\n\ntrain_features = train_data.drop(columns=['sii'])\n\n\n\n# 2. 合并 train_features、valid_data 和 test，并记录每个部分的样本数量\n\nn_train = len(train_features)\n\nn_valid = len(valid_data)\n\nn_test = len(test)\n\n\n\ncombined_data = pd.concat([train_features, valid_data, test], ignore_index=True)\n\n\n\n# 3. 选择数值型列进行填充\n\nnumeric_cols = combined_data.select_dtypes(include=['float64', 'int64', 'int32']).columns\n\n\n\n# 4. 进行缺失值填充\n\nimputed_data = imputer.fit_transform(combined_data[numeric_cols])\n\n\n\n# 5. 将填充后的数据转换为 DataFrame\n\nimputed_df = pd.DataFrame(imputed_data, columns=numeric_cols)\n\n\n\n# 6. 将非数值型列添加回去\n\nfor col in combined_data.columns:\n\n if col not in numeric_cols:\n\n imputed_df[col] = combined_data[col].values\n\n\n\n# 7. 将数据拆分回 train_data, valid_data 和 test\n\ntrain_filled_df = imputed_df.iloc[:n_train].reset_index(drop=True)\n\nvalid_filled_df = imputed_df.iloc[n_train:n_train + n_valid].reset_index(drop=True)\n\ntest_filled_df = imputed_df.iloc[n_train + n_valid:].reset_index(drop=True)\n\n\n\n# 8. 将 'sii' 列添加回 train_filled_df\n\ntrain_filled_df['sii'] = sii_column\n\n\n\n# 9. 更新 train_data, valid_data 和 test\n\ntrain_data = train_filled_df\n\nvalid_data = valid_filled_df\n\ntest = test_filled_df\n\n\n\nprint(\"train_data, valid_data 和 test 缺失值填补完毕\")\n\n\n","metadata":{"papermill":{"duration":2.992632,"end_time":"2024-11-20T15:17:03.536492","exception":false,"start_time":"2024-11-20T15:17:00.543860","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:36.322126Z","iopub.execute_input":"2024-12-12T14:54:36.322439Z","iopub.status.idle":"2024-12-12T14:54:38.968505Z","shell.execute_reply.started":"2024-12-12T14:54:36.322407Z","shell.execute_reply":"2024-12-12T14:54:38.967575Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"对withtime进行数据填充","metadata":{"papermill":{"duration":0.037988,"end_time":"2024-11-20T15:17:03.613348","exception":false,"start_time":"2024-11-20T15:17:03.575360","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer, KNNImputer\n\nfrom sklearn.experimental import enable_iterative_imputer\n\nfrom sklearn.impute import IterativeImputer\n\n# imputer = IterativeImputer(max_iter=10, random_state=0)\n\nimputer = KNNImputer(n_neighbors=5)\n\n\n\n# 对train填充\n\n# 1. 提取 'sii' 列\n\nsii_column = train_data_withtime['sii'].values\n\ntrain_features = train_data_withtime.drop(columns=['sii'])\n\n\n\n# 2. 合并 train_features 和 valid_data_withtime\n\ncombined_data = pd.concat([train_features, valid_data_withtime], ignore_index=True)\n\n\n\n# 3. 选择数值型列进行填充\n\nnumeric_cols = combined_data.select_dtypes(include=['float64', 'int64', 'int32']).columns\n\n\n\n# 4. 进行缺失值填充\n\nimputer = KNNImputer(n_neighbors=5)\n\nimputed_data = imputer.fit_transform(combined_data[numeric_cols])\n\n\n\n# 5. 将填充后的数据转换为 DataFrame\n\nimputed_df = pd.DataFrame(imputed_data, columns=numeric_cols)\n\n\n\n# 6. 将非数值型列添加回去\n\nfor col in combined_data.columns:\n\n if col not in numeric_cols:\n\n imputed_df[col] = combined_data[col].values\n\n\n\n# 7. 将数据拆分回 train_data_withtime 和 valid_data_withtime\n\ntrain_filled_df = imputed_df.iloc[:len(train_features)].reset_index(drop=True)\n\nvalid_filled_df = imputed_df.iloc[len(train_features):].reset_index(drop=True)\n\n\n\n# 8. 将 'sii' 列添加回 train_filled_df\n\ntrain_filled_df['sii'] = sii_column\n\n\n\n# 9. 更新 train_data_withtime 和 valid_data_withtime\n\ntrain_data_withtime = train_filled_df\n\nvalid_data_withtime = valid_filled_df\n\n\n\nprint(\"train_data_withtime 和 valid_data_withtime 缺失值填补完毕\")","metadata":{"papermill":{"duration":0.504443,"end_time":"2024-11-20T15:17:04.155942","exception":false,"start_time":"2024-11-20T15:17:03.651499","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:38.969627Z","iopub.execute_input":"2024-12-12T14:54:38.969911Z","iopub.status.idle":"2024-12-12T14:54:39.428581Z","shell.execute_reply.started":"2024-12-12T14:54:38.969885Z","shell.execute_reply":"2024-12-12T14:54:39.427614Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"填充方法3：使用IterativeImputer填充","metadata":{"papermill":{"duration":0.038077,"end_time":"2024-11-20T15:17:04.233156","exception":false,"start_time":"2024-11-20T15:17:04.195079","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from sklearn.impute import SimpleImputer, KNNImputer\n\n# from sklearn.experimental import enable_iterative_imputer\n\n# from sklearn.impute import IterativeImputer\n\n# imputer = IterativeImputer(max_iter=10, random_state=0)\n\n\n\n# # 对train填充\n\n# # 1. 提取 'sii' 列\n\n# sii_column = train_data['sii'].values\n\n# train_features = train_data.drop(columns=['sii'])\n\n\n\n# # 2. 合并 train_features 和 valid_data\n\n# combined_data = pd.concat([train_features, valid_data], ignore_index=True)\n\n\n\n# # 3. 选择数值型列进行填充\n\n# numeric_cols = combined_data.select_dtypes(include=['float64', 'int64', 'int32']).columns\n\n\n\n# # 4. 进行缺失值填充\n\n# imputer = KNNImputer(n_neighbors=10)\n\n# imputed_data = imputer.fit_transform(combined_data[numeric_cols])\n\n\n\n# # 5. 将填充后的数据转换为 DataFrame\n\n# imputed_df = pd.DataFrame(imputed_data, columns=numeric_cols)\n\n\n\n# # 6. 将非数值型列添加回去\n\n# for col in combined_data.columns:\n\n# if col not in numeric_cols:\n\n# imputed_df[col] = combined_data[col].values\n\n\n\n# # 7. 将数据拆分回 train_data 和 valid_data\n\n# train_filled_df = imputed_df.iloc[:len(train_features)].reset_index(drop=True)\n\n# valid_filled_df = imputed_df.iloc[len(train_features):].reset_index(drop=True)\n\n\n\n# # 8. 将 'sii' 列添加回 train_filled_df\n\n# train_filled_df['sii'] = sii_column\n\n\n\n# # 9. 更新 train_data 和 valid_data\n\n# train_data = train_filled_df\n\n# valid_data = valid_filled_df\n\n\n\n# print(\"train_data 和 valid_data 缺失值填补完毕\")","metadata":{"papermill":{"duration":0.045938,"end_time":"2024-11-20T15:17:04.317328","exception":false,"start_time":"2024-11-20T15:17:04.271390","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.429982Z","iopub.execute_input":"2024-12-12T14:54:39.430444Z","iopub.status.idle":"2024-12-12T14:54:39.436100Z","shell.execute_reply.started":"2024-12-12T14:54:39.430388Z","shell.execute_reply":"2024-12-12T14:54:39.435255Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"填充方法4：使用GAIN填充","metadata":{"papermill":{"duration":0.037997,"end_time":"2024-11-20T15:17:04.394188","exception":false,"start_time":"2024-11-20T15:17:04.356191","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 定义归一化和反归一化函数\n\ndef min_max_normalize(data):\n\n col_mins = np.nanmin(data, axis=0)\n\n col_maxs = np.nanmax(data, axis=0)\n\n # 防止除以零\n\n col_ranges = col_maxs - col_mins\n\n col_ranges[col_ranges == 0] = 1\n\n normalized_data = (data - col_mins) / col_ranges\n\n return normalized_data, col_mins, col_ranges\n\n\n\ndef min_max_inverse_transform(normalized_data, col_mins, col_ranges):\n\n data = normalized_data * col_ranges + col_mins\n\n return data","metadata":{"papermill":{"duration":0.046385,"end_time":"2024-11-20T15:17:04.478836","exception":false,"start_time":"2024-11-20T15:17:04.432451","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.437216Z","iopub.execute_input":"2024-12-12T14:54:39.437471Z","iopub.status.idle":"2024-12-12T14:54:39.448600Z","shell.execute_reply.started":"2024-12-12T14:54:39.437446Z","shell.execute_reply":"2024-12-12T14:54:39.447741Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import MinMaxScaler\n\nclass GAIN:\n def __init__(self, data_dim, hint_rate=0.9, alpha=10, epochs=1000, batch_size=128, learning_rate=0.001):\n self.data_dim = data_dim\n self.hint_rate = hint_rate\n self.alpha = alpha\n self.epochs = epochs\n self.batch_size = batch_size\n self.learning_rate = learning_rate\n self.build_model()\n\n def build_model(self):\n self.X = tf.keras.layers.Input(shape=(self.data_dim,))\n self.M = tf.keras.layers.Input(shape=(self.data_dim,), dtype=tf.float32) # Mask matrix\n self.H = tf.keras.layers.Input(shape=(self.data_dim,), dtype=tf.float32) # Hint matrix\n\n # Generator\n G_input = tf.keras.layers.Concatenate()([self.X, self.M])\n G_hidden = tf.keras.layers.Dense(units=64, activation='relu')(G_input)\n self.G_output = tf.keras.layers.Dense(units=self.data_dim, activation='sigmoid')(G_hidden)\n\n # Discriminator\n D_input = tf.keras.layers.Concatenate()([self.X * self.M + self.G_output * (1 - self.M), self.H])\n D_hidden = tf.keras.layers.Dense(units=64, activation='relu')(D_input)\n self.D_output = tf.keras.layers.Dense(units=self.data_dim, activation='sigmoid')(D_hidden)\n\n # 创建生成器和判别器模型\n self.generator = tf.keras.models.Model(inputs=[self.X, self.M], outputs=self.G_output)\n self.discriminator = tf.keras.models.Model(inputs=[self.X, self.M, self.H], outputs=self.D_output)\n\n def train(self, data_x, missing_mask):\n epsilon = 1e-8 # 用于防止 log 操作中的 NaN\n\n # 每次训练时重新初始化优化器，确保适应新变量\n self.optimizer = tf.keras.optimizers.Adam(learning_rate=self.learning_rate)\n\n for epoch in range(self.epochs):\n # 随机抽取一个批次的数据\n batch_idx = np.random.choice(len(data_x), self.batch_size, replace=False)\n X_batch = data_x[batch_idx]\n M_batch = missing_mask[batch_idx]\n\n # 随机生成提示矩阵\n H_batch = M_batch * np.random.binomial(1, self.hint_rate, M_batch.shape)\n\n with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n G_output = self.generator([X_batch, M_batch])\n D_output = self.discriminator([X_batch, M_batch, H_batch])\n\n # 计算损失，使用 epsilon 防止 NaN\n D_loss = -tf.reduce_mean(\n M_batch * tf.math.log(D_output + epsilon) + (1 - M_batch) * tf.math.log(1 - D_output + epsilon)\n )\n G_loss = -tf.reduce_mean((1 - M_batch) * tf.math.log(D_output + epsilon)) + \\\n self.alpha * tf.reduce_mean((M_batch * X_batch - M_batch * G_output) ** 2)\n\n # 计算梯度\n gen_gradients = gen_tape.gradient(G_loss, self.generator.trainable_variables)\n disc_gradients = disc_tape.gradient(D_loss, self.discriminator.trainable_variables)\n\n # 应用梯度更新\n self.optimizer.apply_gradients(zip(gen_gradients, self.generator.trainable_variables))\n self.optimizer.apply_gradients(zip(disc_gradients, self.discriminator.trainable_variables))\n\n if epoch % 100 == 0:\n print(f'Epoch {epoch}, D_loss: {D_loss.numpy():.4f}, G_loss: {G_loss.numpy():.4f}')\n\n # 最终填充缺失值\n filled_data = self.generator([data_x, missing_mask])\n imputed_data = data_x * missing_mask + filled_data * (1 - missing_mask)\n return imputed_data","metadata":{"papermill":{"duration":10.872504,"end_time":"2024-11-20T15:17:15.390388","exception":false,"start_time":"2024-11-20T15:17:04.517884","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.449638Z","iopub.execute_input":"2024-12-12T14:54:39.449924Z","iopub.status.idle":"2024-12-12T14:54:39.465673Z","shell.execute_reply.started":"2024-12-12T14:54:39.449883Z","shell.execute_reply":"2024-12-12T14:54:39.464814Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # 1. 提取 'sii' 列\n\n# sii_column = train_data['sii'].values\n\n# train_features = train_data.drop(columns=['sii'])\n\n\n\n# # 2. 合并 train_features 和 valid_data\n\n# combined_data = pd.concat([train_features, valid_data], ignore_index=True)\n\n\n\n# # 3. 转换为 numpy 数组\n\n# combined_values = combined_data.values\n\n\n\n# # 4. 创建缺失值掩码\n\n\n\n# nan_mask = np.isnan(combined_values)\n\n\n\n# # 5. 归一化数据\n\n# normalized_data, col_mins, col_ranges = min_max_normalize(combined_values)\n\n\n\n# # 6. 使用 GAIN 填补缺失值\n\n# data_dim = normalized_data.shape[1]\n\n# gain = GAIN(data_dim)\n\n\n\n# # 将数据类型转换为 float32，以适应 TensorFlow 的数据类型要求\n\n# normalized_data = normalized_data.astype(np.float32)\n\n# nan_mask = nan_mask.astype(np.float32)\n\n\n\n# # 填补缺失值\n\n# imputed_data_normalized = gain.train(normalized_data, nan_mask)\n\n\n\n# # 7. 逆归一化回原始尺度\n\n# imputed_data_combined = min_max_inverse_transform(imputed_data_normalized, col_mins, col_ranges)\n\n\n\n# # 8. 将数据拆分回 train_data 和 valid_data\n\n# imputed_data_train = imputed_data_combined[:len(train_features)]\n\n# imputed_data_valid = imputed_data_combined[len(train_features):]\n\n\n\n# # 9. 创建 DataFrame\n\n# train_filled_df = pd.DataFrame(imputed_data_train, columns=train_features.columns)\n\n# valid_filled_df = pd.DataFrame(imputed_data_valid, columns=valid_data.columns)\n\n\n\n# # 10. 将 'sii' 列添加回 train_filled_df\n\n# train_filled_df['sii'] = sii_column\n\n\n\n# # 11. 更新 train_data 和 valid_data\n\n# train_data = train_filled_df\n\n# valid_data = valid_filled_df\n\n\n\n# # 12. 保存结果（可选）\n\n# train_filled_df.to_csv(\"train_filled.csv\", index=False)\n\n# valid_filled_df.to_csv(\"valid_filled.csv\", index=False)\n\n\n\n# print(\"train_data 和 valid_data 缺失值填补完毕\")","metadata":{"papermill":{"duration":0.046556,"end_time":"2024-11-20T15:17:15.475775","exception":false,"start_time":"2024-11-20T15:17:15.429219","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.466952Z","iopub.execute_input":"2024-12-12T14:54:39.467589Z","iopub.status.idle":"2024-12-12T14:54:39.481957Z","shell.execute_reply.started":"2024-12-12T14:54:39.467549Z","shell.execute_reply":"2024-12-12T14:54:39.481069Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"填充方法5：使用DNN填充","metadata":{"papermill":{"duration":0.039094,"end_time":"2024-11-20T15:17:15.553246","exception":false,"start_time":"2024-11-20T15:17:15.514152","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\n\n# from sklearn.preprocessing import StandardScaler\n\n# from sklearn.impute import SimpleImputer\n\n# import tensorflow as tf\n\n# from tensorflow.keras.layers import Input, Dense, Masking\n\n# from tensorflow.keras.models import Model\n\n# import rando\n\n\n\n# # 1. 提取 'sii' 列\n\n# sii_column = train_data['sii'].values\n\n# train_features = train_data.drop(columns=['sii'])\n\n\n\n# # 2. 合并 train_features 和 valid_data\n\n# combined_data = pd.concat([train_features, valid_data], ignore_index=True)\n\n\n\n# # 3. 转换为 numpy 数组\n\n# combined_values = combined_data.values\n\n\n\n# # 4. 创建缺失值掩码\n\n# nan_mask = np.isnan(combined_values)\n\n\n\n# # 5. 数据标准化（仅对数值型数据）\n\n# numeric_cols = combined_data.select_dtypes(include=['float64', 'int64', 'int32']).columns\n\n# scaler = StandardScaler()\n\n# combined_values_scaled = combined_values.copy()\n\n# combined_values_scaled[:, :] = scaler.fit_transform(SimpleImputer(strategy='mean').fit_transform(combined_values))\n\n\n\n# # 6. 填补 NaN 值（临时填充为 0，便于模型输入）\n\n# combined_values_filled = np.nan_to_num(combined_values_scaled, nan=0.0)\n\n\n\n# # 7. 构建 DNN 模型\n\n# input_dim = combined_values_filled.shape[1]\n\n\n\n# inputs = Input(shape=(input_dim,))\n\n# masking_layer = Masking(mask_value=0.0)(inputs)\n\n# x = Dense(256, activation='relu')(masking_layer)\n\n# x = Dense(128, activation='relu')(x)\n\n# x = Dense(64, activation='relu')(x)\n\n# outputs = Dense(input_dim, activation='linear')(x)\n\n\n\n# model = Model(inputs=inputs, outputs=outputs)\n\n# model.compile(optimizer='adam', loss='mse')\n\n\n\n# # 8. 训练模型（仅在观测值上计算损失）\n\n# def custom_loss(y_true, y_pred):\n\n# mask = tf.cast(tf.not_equal(y_true, 0.0), tf.float32)\n\n# loss = tf.reduce_mean(mask * tf.square(y_true - y_pred))\n\n# return loss\n\n\n\n# model.compile(optimizer='adam', loss=custom_loss)\n\n\n\n# # 训练模型\n\n# model.fit(combined_values_filled, combined_values_filled,\n\n# epochs=50,\n\n# batch_size=64,\n\n# verbose=1)\n\n\n\n# # 9. 使用模型填补缺失值\n\n# imputed_data_scaled = model.predict(combined_values_filled)\n\n\n\n# # 只在原始数据中缺失的位置填充预测值，保留观测值\n\n# imputed_data_scaled = combined_values_filled * (~nan_mask) + imputed_data_scaled * nan_mask\n\n\n\n# # 10. 将数据逆标准化回原始尺度\n\n# imputed_data = scaler.inverse_transform(imputed_data_scaled)\n\n\n\n# # 11. 将数据拆分回 train_data 和 valid_data\n\n# imputed_data_train = imputed_data[:len(train_features)]\n\n# imputed_data_valid = imputed_data[len(train_features):]\n\n\n\n# # 12. 创建 DataFrame\n\n# train_filled_df = pd.DataFrame(imputed_data_train, columns=combined_data.columns)\n\n# valid_filled_df = pd.DataFrame(imputed_data_valid, columns=combined_data.columns)\n\n\n\n# # 13. 将 'sii' 列添加回 train_filled_df\n\n# train_filled_df['sii'] = sii_column\n\n\n\n# # 14. 更新 train_data 和 valid_data\n\n# train_data = train_filled_df\n\n# valid_data = valid_filled_df\n\n\n\n# # 15. 保存结果（可选）\n\n# train_filled_df.to_csv(\"train_filled.csv\", index=False)\n\n# valid_filled_df.to_csv(\"valid_filled.csv\", index=False)\n\n\n\n# print(\"train_data 和 valid_data 缺失值填补完毕\")","metadata":{"papermill":{"duration":0.047538,"end_time":"2024-11-20T15:17:15.639661","exception":false,"start_time":"2024-11-20T15:17:15.592123","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.483260Z","iopub.execute_input":"2024-12-12T14:54:39.483607Z","iopub.status.idle":"2024-12-12T14:54:39.496140Z","shell.execute_reply.started":"2024-12-12T14:54:39.483570Z","shell.execute_reply":"2024-12-12T14:54:39.495267Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"填充方法6，使用AutoKNN填充","metadata":{"papermill":{"duration":0.041076,"end_time":"2024-11-20T15:17:15.719645","exception":false,"start_time":"2024-11-20T15:17:15.678569","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# import numpy as np\n\n# import pandas as pd\n\n# from sklearn.impute import KNNImputer\n\n# from sklearn.model_selection import GridSearchCV\n\n# from sklearn.pipeline import Pipeline\n\n# from sklearn.compose import ColumnTransformer\n\n# from sklearn.preprocessing import StandardScaler\n\n# from sklearn.metrics import make_scorer, mean_squared_error\n\n# from sklearn.base import BaseEstimator, TransformerMixin\n\n\n\n# # 自定义 DataFrameAdapter 保持数据格式和参数兼容性\n\n# class DataFrameAdapter(TransformerMixin, BaseEstimator):\n\n# def __init__(self, transformer):\n\n# self.transformer = transformer\n\n\n\n# def fit(self, X, y=None):\n\n# self.transformer.fit(X, y)\n\n# return self\n\n\n\n# def transform(self, X):\n\n# X_transformed = self.transformer.transform(X)\n\n# if isinstance(X_transformed, np.ndarray):\n\n# return pd.DataFrame(X_transformed, columns=X.columns, index=X.index)\n\n# return X_transformed\n\n\n\n# def get_params(self, deep=True):\n\n# return {\"transformer\": self.transformer}\n\n\n\n# def set_params(self, **params):\n\n# if \"transformer\" in params:\n\n# self.transformer = params[\"transformer\"]\n\n# else:\n\n# self.transformer.set_params(**params)\n\n# return self\n\n\n\n# # 自定义评分函数\n\n# def imputation_score(estimator, X, y=None):\n\n# X_copy = X.copy()\n\n# rng = np.random.RandomState(42)\n\n \n\n# # 随机选取10%的数据作为验证集，并引入缺失值\n\n# missing_mask = rng.rand(*X_copy.shape) < 0.1\n\n# X_missing = X_copy.copy()\n\n# X_missing[missing_mask] = np.nan\n\n \n\n# # 使用填充器对引入缺失值的数据进行填充\n\n# X_imputed = estimator.transform(X_missing)\n\n \n\n# # 计算填充值与原始真实值之间的均方误差\n\n# mse = mean_squared_error(X_copy[missing_mask], X_imputed[missing_mask])\n\n# return -mse\n\n\n\n# # 创建评分器\n\n# imputation_scorer = make_scorer(imputation_score, greater_is_better=True)\n\n\n\n# # 示例数据的准备（假设 train_data 和 valid_data 是输入的数据集）\n\n# # Step 1: 提取 'sii' 列\n\n# sii_column = train_data['sii'].values\n\n# train_features = train_data.drop(columns=['sii'])\n\n\n\n# # Step 2: 合并 train_features 和 valid_data\n\n# combined_data = pd.concat([train_features, valid_data], ignore_index=True)\n\n\n\n# # Step 3: 选择数值型列进行填充\n\n# numeric_cols = combined_data.select_dtypes(include=['float64', 'int64', 'int32']).columns\n\n\n\n# # Step 4: 定义KNNImputer和参数网格\n\n# param_grid = {\n\n# 'transformer__num__imputer__n_neighbors': [1, 2, 3, 4, 5]\n\n# }\n\n\n\n# # Step 5: 构建预处理管道\n\n# numeric_transformer = Pipeline(steps=[\n\n# ('scaler', StandardScaler()), # 标准化数据\n\n# ('imputer', KNNImputer(metric='nan_euclidean')) # 使用 KNNImputer 进行填充\n\n# ])\n\n\n\n# preprocessor = ColumnTransformer(\n\n# transformers=[\n\n# ('num', numeric_transformer, numeric_cols)\n\n# ]\n\n# )\n\n\n\n# # 将 ColumnTransformer 包装在 DataFrameAdapter 中\n\n# preprocessor_adapter = DataFrameAdapter(preprocessor)\n\n\n\n# # Step 6: 使用 GridSearchCV 进行参数调优\n\n# grid_search = GridSearchCV(\n\n# preprocessor_adapter,\n\n# param_grid=param_grid,\n\n# cv=3, # 三折交叉验证\n\n# scoring=imputation_scorer, # 使用自定义评分函数\n\n# verbose=1\n\n# )\n\n\n\n# # Step 7: 在 combined_data 上进行调参和填充\n\n# grid_search.fit(combined_data[numeric_cols])\n\n\n\n# # 使用最佳参数进行填充\n\n# best_imputer = grid_search.best_estimator_\n\n# imputed_data = best_imputer.transform(combined_data[numeric_cols])\n\n\n\n# # Step 8: 将填充后的数据转换为 DataFrame\n\n# imputed_df = pd.DataFrame(imputed_data, columns=numeric_cols)\n\n\n\n# # Step 9: 将非数值型列添加回去\n\n# for col in combined_data.columns:\n\n# if col not in numeric_cols:\n\n# imputed_df[col] = combined_data[col].values\n\n\n\n# # Step 10: 将数据拆分回 train_data 和 valid_data\n\n# train_filled_df = imputed_df.iloc[:len(train_features)].reset_index(drop=True)\n\n# valid_filled_df = imputed_df.iloc[len(train_features):].reset_index(drop=True)\n\n\n\n# # Step 11: 将 'sii' 列添加回 train_filled_df\n\n# train_filled_df['sii'] = sii_column\n\n\n\n# # Step 12: 更新 train_data 和 valid_data\n\n# train_data = train_filled_df\n\n# valid_data = valid_filled_df\n\n\n\n# print(\"train_data 和 valid_data 缺失值填补完毕，使用 AutoKNN 选择的最佳参数：\", grid_search.best_params_)\n\n\n","metadata":{"papermill":{"duration":0.050069,"end_time":"2024-11-20T15:17:15.809092","exception":false,"start_time":"2024-11-20T15:17:15.759023","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.497675Z","iopub.execute_input":"2024-12-12T14:54:39.497959Z","iopub.status.idle":"2024-12-12T14:54:39.508565Z","shell.execute_reply.started":"2024-12-12T14:54:39.497932Z","shell.execute_reply":"2024-12-12T14:54:39.507554Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"填充方法7：使用WKNN进行填充","metadata":{"papermill":{"duration":0.038946,"end_time":"2024-11-20T15:17:15.886933","exception":false,"start_time":"2024-11-20T15:17:15.847987","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# import numpy as np\n\n# import pandas as pd\n\n# from sklearn.base import BaseEstimator, TransformerMixin\n\n# from sklearn.pipeline import Pipeline\n\n# from sklearn.compose import ColumnTransformer\n\n# from sklearn.preprocessing import StandardScaler\n\n\n\n# # 自定义加权KNN填充器\n\n# class WKNNImputer(BaseEstimator, TransformerMixin):\n\n# def __init__(self, n_neighbors=10):\n\n# self.n_neighbors = n_neighbors\n\n\n\n# def fit(self, X, y=None):\n\n# self.X_filled = X.copy() # 复制数据集用于填充\n\n# self.mask = np.isnan(X) # 找到缺失值的位置\n\n# return self\n\n\n\n# def transform(self, X):\n\n# X_filled = self.X_filled\n\n# mask = self.mask\n\n\n\n# # 对每个缺失值进行加权KNN填充\n\n# for i in range(X.shape[0]):\n\n# for j in range(X.shape[1]):\n\n# if mask[i, j]:\n\n# # 计算该样本的距离，并找到邻居\n\n# distances = np.linalg.norm(X_filled[~mask[:, j], :] - X_filled[i, :], axis=1)\n\n# nearest_indices = np.argsort(distances)[:self.n_neighbors]\n\n# nearest_distances = distances[nearest_indices]\n\n# nearest_values = X_filled[~mask[:, j], j][nearest_indices]\n\n\n\n# # 使用距离的倒数作为权重进行加权填充\n\n# weights = 1 / (nearest_distances + 1e-5)\n\n# X_filled[i, j] = np.sum(weights * nearest_values) / np.sum(weights)\n\n\n\n# return X_filled\n\n\n\n# # 示例数据的准备（假设 train_data 和 valid_data 是输入的数据集）\n\n# # Step 1: 提取 'sii' 列\n\n# sii_column = train_data['sii'].values\n\n# train_features = train_data.drop(columns=['sii'])\n\n\n\n# # Step 2: 合并 train_features 和 valid_data\n\n# combined_data = pd.concat([train_features, valid_data], ignore_index=True)\n\n\n\n# # Step 3: 选择数值型列进行填充\n\n# numeric_cols = combined_data.select_dtypes(include=['float64', 'int64', 'int32']).columns\n\n\n\n# # Step 4: 构建预处理管道\n\n# numeric_transformer = Pipeline(steps=[\n\n# ('scaler', StandardScaler()), # 标准化数据\n\n# ('imputer', WKNNImputer(n_neighbors=10)) # 使用自定义的 WKNNImputer 进行加权填充，邻居数为5\n\n# ])\n\n\n\n# preprocessor = ColumnTransformer(\n\n# transformers=[\n\n# ('num', numeric_transformer, numeric_cols)\n\n# ]\n\n# )\n\n\n\n# # Step 5: 在 combined_data 上进行填充\n\n# imputed_data = preprocessor.fit_transform(combined_data[numeric_cols])\n\n\n\n# # Step 6: 将填充后的数据转换为 DataFrame\n\n# imputed_df = pd.DataFrame(imputed_data, columns=numeric_cols)\n\n\n\n# # Step 7: 将非数值型列添加回去\n\n# for col in combined_data.columns:\n\n# if col not in numeric_cols:\n\n# imputed_df[col] = combined_data[col].values\n\n\n\n# # Step 8: 将数据拆分回 train_data 和 valid_data\n\n# train_filled_df = imputed_df.iloc[:len(train_features)].reset_index(drop=True)\n\n# valid_filled_df = imputed_df.iloc[len(train_features):].reset_index(drop=True)\n\n\n\n# # Step 9: 将 'sii' 列添加回 train_filled_df\n\n# train_filled_df['sii'] = sii_column\n\n\n\n# # Step 10: 更新 train_data 和 valid_data\n\n# train_data = train_filled_df\n\n# valid_data = valid_filled_df\n\n\n\n# print(\"train_data 和 valid_data 缺失值填补完毕，使用固定的 Weighted KNN 进行填充。\")\n\n\n","metadata":{"papermill":{"duration":0.047914,"end_time":"2024-11-20T15:17:15.973430","exception":false,"start_time":"2024-11-20T15:17:15.925516","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.509811Z","iopub.execute_input":"2024-12-12T14:54:39.510123Z","iopub.status.idle":"2024-12-12T14:54:39.520271Z","shell.execute_reply.started":"2024-12-12T14:54:39.510085Z","shell.execute_reply":"2024-12-12T14:54:39.519388Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"train shape: {train_data.shape}\")\n\nprint(\"train 中的缺失值数目：\")\n\nprint(train_data.isnull().sum())\n\n\n\n\n\nprint(f\"valid shape: {valid_data.shape}\")\n\nprint(\"valid 中的缺失值数目：\")\n\nprint(valid_data.isnull().sum())\n\n\n\nprint(f\"train_withtime shape: {train_data_withtime.shape}\")\n\nprint(\"train_withtime 中的缺失值数目：\")\n\nprint(train_data_withtime.isnull().sum())\n\n\n\n\n\nprint(f\"valid_withtime shape: {valid_data_withtime.shape}\")\n\nprint(\"valid 中的缺失值数目：\")\n\nprint(valid_data_withtime.isnull().sum())","metadata":{"papermill":{"duration":0.053018,"end_time":"2024-11-20T15:17:16.065075","exception":false,"start_time":"2024-11-20T15:17:16.012057","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.521609Z","iopub.execute_input":"2024-12-12T14:54:39.522556Z","iopub.status.idle":"2024-12-12T14:54:39.539571Z","shell.execute_reply.started":"2024-12-12T14:54:39.522526Z","shell.execute_reply":"2024-12-12T14:54:39.538587Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"过采样方法1：RandomOverSampler","metadata":{"papermill":{"duration":0.038643,"end_time":"2024-11-20T15:17:16.142912","exception":false,"start_time":"2024-11-20T15:17:16.104269","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from imblearn.over_sampling import RandomOverSampler\n\n# import pandas as pd\n\n\n\n# # 1. 将特征和目标变量分开\n\n# X_train = train_data.drop(columns=['sii'])\n\n# y_train = train_data['sii']\n\n\n\n# # 2. 初始化 RandomOverSampler\n\n# oversampler = RandomOverSampler(random_state=42)\n\n\n\n# # 3. 进行过采样\n\n# X_resampled, y_resampled = oversampler.fit_resample(X_train, y_train)\n\n\n\n# # 4. 将过采样后的数据组合成新的 DataFrame\n\n# train_data_resampled = pd.DataFrame(X_resampled, columns=X_train.columns)\n\n# train_data_resampled['sii'] = y_resampled\n\n\n\n# # 5. 输出过采样后的数据集信息\n\n# print(\"过采样后的每类样本数量：\")\n\n# print(train_data_resampled['sii'].value_counts())\n\n\n\n# # 如果需要，您可以将 train_data 更新为过采样后的数据集\n\n# train_data = train_data_resampled\n\n\n\n# print(f\"train shape: {train_data.shape}\")\n\n# print(\"train 中的缺失值数目：\")\n\n# print(train_data.isnull().sum())\n\n\n\n\n\n# print(f\"valid shape: {valid_data.shape}\")\n\n# print(\"valid 中的缺失值数目：\")\n\n# print(valid_data.isnull().sum())","metadata":{"papermill":{"duration":0.047229,"end_time":"2024-11-20T15:17:16.229482","exception":false,"start_time":"2024-11-20T15:17:16.182253","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.540788Z","iopub.execute_input":"2024-12-12T14:54:39.541152Z","iopub.status.idle":"2024-12-12T14:54:39.546400Z","shell.execute_reply.started":"2024-12-12T14:54:39.541114Z","shell.execute_reply":"2024-12-12T14:54:39.545508Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"过采样方法2：KMeans-SMOTE","metadata":{"papermill":{"duration":0.038726,"end_time":"2024-11-20T15:17:16.307233","exception":false,"start_time":"2024-11-20T15:17:16.268507","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from imblearn.over_sampling import KMeansSMOTE\n\n# import pandas as pd\n\n\n\n# # 1. 将特征和目标变量分开\n\n# X_train = train_data.drop(columns=['sii']).copy()\n\n# y_train = train_data['sii'].copy()\n\n\n\n# # 2. 初始化 KMeansSMOTE\n\n# kmeans_smote = KMeansSMOTE(random_state=42, cluster_balance_threshold=0.1)\n\n\n\n# # 3. 进行过采样\n\n# X_resampled, y_resampled = kmeans_smote.fit_resample(X_train, y_train)\n\n\n\n# # 4. 将过采样后的数据组合成新的 DataFrame\n\n# train_data_resampled = pd.DataFrame(X_resampled, columns=X_train.columns)\n\n# train_data_resampled['sii'] = y_resampled\n\n\n\n# # 5. 输出过采样后的数据集信息\n\n# print(\"过采样后的每类样本数量：\")\n\n# print(train_data_resampled['sii'].value_counts())\n\n\n\n# # 更新 train_data 为过采样后的数据集\n\n# train_data = train_data_resampled\n\n\n\n# print(f\"train shape: {train_data.shape}\")\n\n# print(\"train 中的缺失值数目：\")\n\n# print(train_data.isnull().sum())\n\n\n\n# print(f\"valid shape: {valid_data.shape}\")\n\n# print(\"valid 中的缺失值数目：\")\n\n# print(valid_data.isnull().sum())\n","metadata":{"papermill":{"duration":0.049295,"end_time":"2024-11-20T15:17:16.395471","exception":false,"start_time":"2024-11-20T15:17:16.346176","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.547441Z","iopub.execute_input":"2024-12-12T14:54:39.547695Z","iopub.status.idle":"2024-12-12T14:54:39.561568Z","shell.execute_reply.started":"2024-12-12T14:54:39.547669Z","shell.execute_reply":"2024-12-12T14:54:39.560826Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"过采样方法3：Fe-SMOTE","metadata":{"papermill":{"duration":0.037427,"end_time":"2024-11-20T15:17:16.474157","exception":false,"start_time":"2024-11-20T15:17:16.436730","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from imblearn.over_sampling import SMOTE\n\n# from sklearn.feature_selection import SelectKBest, f_classif\n\n# import pandas as pd\n\n\n\n# # 1. 将特征和目标变量分开\n\n# X_train = train_data.drop(columns=['sii']).copy()\n\n# y_train = train_data['sii'].copy()\n\n\n\n# # 2. 使用 SelectKBest 选择最重要的特征\n\n# k_best = 10 # 选择你需要的特征数量\n\n# selector = SelectKBest(score_func=f_classif, k=k_best)\n\n# X_selected = selector.fit_transform(X_train, y_train)\n\n\n\n# # 3. 初始化 SMOTE\n\n# smote = SMOTE(random_state=42)\n\n\n\n# # 4. 对选择的特征进行过采样\n\n# X_resampled_selected, y_resampled = smote.fit_resample(X_selected, y_train)\n\n\n\n# # 5. 将过采样后的数据转换为 DataFrame\n\n# X_resampled_selected_df = pd.DataFrame(X_resampled_selected, columns=X_train.columns[selector.get_support(indices=True)])\n\n\n\n# # 6. 将原始的未选择特征与选择的特征合并\n\n# X_resampled_full = X_train.copy() # 创建原始特征的副本\n\n# X_resampled_full = X_resampled_full.reindex(X_resampled_selected_df.index) # 调整行索引\n\n# X_resampled_full[X_resampled_selected_df.columns] = X_resampled_selected_df # 仅替换选择的特征\n\n\n\n# # 7. 将过采样后的数据组合成新的 DataFrame\n\n# train_data_resampled = X_resampled_full.copy()\n\n# train_data_resampled['sii'] = y_resampled\n\n\n\n# # 8. 输出过采样后的数据集信息\n\n# print(\"过采样后的每类样本数量：\")\n\n# print(train_data_resampled['sii'].value_counts())\n\n\n\n# # 更新 train_data 为过采样后的数据集\n\n# train_data = train_data_resampled\n\n\n\n# print(f\"train shape: {train_data.shape}\")\n\n# print(\"train 中的缺失值数目：\")\n\n# print(train_data.isnull().sum())\n\n\n\n# print(f\"valid shape: {valid_data.shape}\")\n\n# print(\"valid 中的缺失值数目：\")\n\n# print(valid_data.isnull().sum())\n\n\n","metadata":{"papermill":{"duration":0.04681,"end_time":"2024-11-20T15:17:16.560014","exception":false,"start_time":"2024-11-20T15:17:16.513204","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.562690Z","iopub.execute_input":"2024-12-12T14:54:39.563228Z","iopub.status.idle":"2024-12-12T14:54:39.573292Z","shell.execute_reply.started":"2024-12-12T14:54:39.563200Z","shell.execute_reply":"2024-12-12T14:54:39.572438Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"过采样方法4：SMOTE","metadata":{"papermill":{"duration":0.038298,"end_time":"2024-11-20T15:17:16.637600","exception":false,"start_time":"2024-11-20T15:17:16.599302","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from imblearn.over_sampling import SMOTE\n\n# import pandas as pd\n\n\n\n# # 1. 将特征和目标变量分开\n\n# X_train = train_data.drop(columns=['sii'])\n\n# y_train = train_data['sii']\n\n\n\n# # 2. 初始化 SMOTE\n\n# oversampler = SMOTE(random_state=42)\n\n\n\n# # 3. 进行过采样\n\n# X_resampled, y_resampled = oversampler.fit_resample(X_train, y_train)\n\n\n\n# # 4. 将过采样后的数据组合成新的 DataFrame\n\n# train_data_resampled = pd.DataFrame(X_resampled, columns=X_train.columns)\n\n# train_data_resampled['sii'] = y_resampled\n\n\n\n# # 5. 输出过采样后的数据集信息\n\n# print(\"过采样后的每类样本数量：\")\n\n# print(train_data_resampled['sii'].value_counts())\n\n\n\n# # 如果需要，您可以将 train_data 更新为过采样后的数据集\n\n# train_data = train_data_resampled\n\n\n\n# print(f\"train shape: {train_data.shape}\")\n\n# print(\"train 中的缺失值数目：\")\n\n# print(train_data.isnull().sum())\n\n\n\n# print(f\"valid shape: {valid_data.shape}\")\n\n# print(\"valid 中的缺失值数目：\")\n\n# print(valid_data.isnull().sum())\n","metadata":{"papermill":{"duration":0.047916,"end_time":"2024-11-20T15:17:16.724005","exception":false,"start_time":"2024-11-20T15:17:16.676089","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.574369Z","iopub.execute_input":"2024-12-12T14:54:39.574709Z","iopub.status.idle":"2024-12-12T14:54:39.584784Z","shell.execute_reply.started":"2024-12-12T14:54:39.574672Z","shell.execute_reply":"2024-12-12T14:54:39.584027Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"使用特征工程","metadata":{"papermill":{"duration":0.039009,"end_time":"2024-11-20T15:17:16.801406","exception":false,"start_time":"2024-11-20T15:17:16.762397","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# train_data = feature_engineering(train_data)\n\n# valid_data = feature_engineering(valid_data)\n# test = feature_engineering(test)\n\n\n\n# print(f\"train shape: {train_data.shape}\")\n\n# print(\"train 中的缺失值数目：\")\n\n# print(train_data.isnull().sum())\n\n\n\n\n\n# print(f\"valid shape: {valid_data.shape}\")\n\n# print(\"valid 中的缺失值数目：\")\n\n# print(valid_data.isnull().sum())\n\n\n\n\n\n# train_data_withtime = feature_engineering(train_data_withtime)\n\n# valid_data_withtime = feature_engineering(valid_data_withtime)\n\n\n\n# print(f\"train_withtime shape: {train_data_withtime.shape}\")\n\n# print(\"train_withtime 中的缺失值数目：\")\n\n# print(train_data_withtime.isnull().sum())\n\n\n\n\n\n# print(f\"valid shape_withtime: {valid_data_withtime.shape}\")\n\n# print(\"valid_withtime 中的缺失值数目：\")\n\n# print(valid_data_withtime.isnull().sum())\n\n\n\n# test = feature_engineering(test)\n\n# print(\"test 中的缺失值数目：\")\n\n# print(test.isnull().sum())","metadata":{"papermill":{"duration":0.049472,"end_time":"2024-11-20T15:17:16.890780","exception":false,"start_time":"2024-11-20T15:17:16.841308","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.585824Z","iopub.execute_input":"2024-12-12T14:54:39.586176Z","iopub.status.idle":"2024-12-12T14:54:39.594810Z","shell.execute_reply.started":"2024-12-12T14:54:39.586140Z","shell.execute_reply":"2024-12-12T14:54:39.593923Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"定义QWK分数计算方式","metadata":{"papermill":{"duration":0.038062,"end_time":"2024-11-20T15:17:16.967921","exception":false,"start_time":"2024-11-20T15:17:16.929859","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def calculate_qwk(y_true, y_pred):\n\n # 将预测值四舍五入为整数类型\n\n y_pred_rounded = np.round(y_pred).astype(int)\n\n y_true = y_true.astype(int)\n\n # 计算 QWK 分数\n\n kappa = cohen_kappa_score(y_true, y_pred_rounded, weights='quadratic')\n\n return kappa","metadata":{"papermill":{"duration":0.046228,"end_time":"2024-11-20T15:17:17.052897","exception":false,"start_time":"2024-11-20T15:17:17.006669","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.595892Z","iopub.execute_input":"2024-12-12T14:54:39.596168Z","iopub.status.idle":"2024-12-12T14:54:39.604459Z","shell.execute_reply.started":"2024-12-12T14:54:39.596143Z","shell.execute_reply":"2024-12-12T14:54:39.603705Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Optuna超参数调优","metadata":{"papermill":{"duration":0.038255,"end_time":"2024-11-20T15:17:17.130047","exception":false,"start_time":"2024-11-20T15:17:17.091792","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\n\n# import numpy as np\n\n# import pandas as pd\n\n# import optuna\n\n# from catboost import CatBoostClassifier, Pool\n\n# from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n\n# from sklearn.model_selection import StratifiedKFold\n\n# from tqdm import tqdm\n\n# import matplotlib.pyplot as plt\n\n# import seaborn as sns\n\n# from scipy.optimize import minimize\n\n# from sklearn.metrics import cohen_kappa_score\n\n# from colorama import Fore, Style\n\n\n\n# # 定义 QWK 计算函数\n\n# def calculate_qwk(y_true, y_pred):\n\n# return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n\n\n# # 设置参数\n\n# n_splits = 5\n\n# SEED = 42\n\n# num_classes = train_data['sii'].nunique()\n\n# class_numbers = np.arange(num_classes)\n\n# train_S = []\n\n# test_S = []\n\n# optuna_s = []\n\n\n\n# # 初始化交叉验证\n\n# SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state = SEED)\n\n\n\n# # 定义 Optuna 调参的目标函数\n\n# def objective(trial):\n\n# params = {\n\n# 'iterations': trial.suggest_int('iterations', 500, 2000),\n\n# 'learning_rate': trial.suggest_loguniform('learning_rate', 1e-3, 0.1),\n\n# 'depth': trial.suggest_int('depth', 4, 10),\n\n# 'l2_leaf_reg': trial.suggest_loguniform('l2_leaf_reg', 1e-3, 10),\n\n# 'random_seed': SEED,\n\n# 'loss_function': 'MultiClass',\n\n# 'eval_metric': 'TotalF1',\n\n# 'verbose': False,\n\n# 'task_type': 'GPU', # 使用 GPU\n\n# 'devices': '0' # 指定 GPU 设备\n\n# }\n\n \n\n \n\n# for fold, (train_idx, val_idx) in enumerate(SKF.split(train_data, train_data['sii'])):\n\n# X_train_cv, X_val_cv = train_data.iloc[train_idx].drop(columns=['sii']), train_data.iloc[val_idx].drop(columns=['sii'])\n\n# y_train_cv, y_val_cv = train_data.iloc[train_idx]['sii'], train_data.iloc[val_idx]['sii']\n\n \n\n# # 删除 'id' 列，防止泄露\n\n# if 'id' in X_train_cv.columns:\n\n# X_train_cv = X_train_cv.drop(columns=['id'])\n\n# if 'id' in X_val_cv.columns:\n\n# X_val_cv = X_val_cv.drop(columns=['id'])\n\n \n\n# # 创建 Pool 对象\n\n# train_pool = Pool(X_train_cv, label=y_train_cv)\n\n# val_pool = Pool(X_val_cv, label=y_val_cv)\n\n \n\n# # 训练模型\n\n# model = CatBoostClassifier(**params)\n\n# model.fit(train_pool, eval_set=val_pool, early_stopping_rounds=50, verbose=False)\n\n \n\n# y_val_pred_proba = model.predict_proba(X_val_cv)\n\n# y_val_pred_continuous = np.dot(y_val_pred_proba, class_numbers)\n\n \n\n# oof_non_rounded[val_idx] = y_val_pred_continuous\n\n# y_val_pred_rounded = np.round(y_val_pred_continuous).astype(int)\n\n \n\n# accuracy = accuracy_score(y_val_cv, y_val_pred_rounded)\n\n# qwk = calculate_qwk(y_val_cv, y_val_pred_rounded)\n\n# print(f\"Fold {fold+1} - Accuracy: {accuracy:.4f}, QWK: {qwk:.4f}\")\n\n \n\n# optuna_s.append(qwk)\n\n \n\n# X_test_cv = valid_data.drop(columns=['id']) if 'id' in valid_data.columns else valid_data.copy()\n\n# y_test_pred_proba = model.predict_proba(X_test_cv)\n\n# y_test_pred_continuous = np.dot(y_test_pred_proba, class_numbers)\n\n# test_preds[:, fold] = y_test_pred_continuous\n\n \n\n# # 返回平均 QWK 作为目标\n\n# return -np.mean(optuna_s) # Optuna 以最小化目标函数为目标，因此返回负的 QWK\n\n\n\n# # 创建 Optuna 学习器\n\n# study = optuna.create_study(direction='minimize')\n\n# study.optimize(objective, n_trials=50)\n\n\n\n# print(f\"Best trial parameters: {study.best_trial.params}\")\n\n# print(f\"Best trial QWK: {-study.best_value:.4f}\")\n\n\n\n# # 使用最佳参数进行训练和预测\n\n# best_params = study.best_trial.params\n\n# best_params.update({\n\n# 'loss_function': 'MultiClass',\n\n# 'random_seed': SEED,\n\n# 'verbose': False,\n\n# 'task_type': 'GPU',\n\n# 'devices': '0'\n\n# })\n\n\n\n# # 在整个数据集上重新训练模型并进行预测\n\n# oof_non_rounded = np.zeros(len(train_data), dtype=float)\n\n# test_preds = np.zeros((len(valid_data), n_splits))\n\n# SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state = SEED)\n\n# for fold, (train_idx, val_idx) in enumerate(tqdm(SKF.split(train_data, train_data['sii']), desc=\"Training Folds\", total=n_splits)):\n\n# print(f\"Fold {fold+1}\")\n\n \n\n# X_train_cv, X_val_cv = train_data.iloc[train_idx].drop(columns=['sii']), train_data.iloc[val_idx].drop(columns=['sii'])\n\n# y_train_cv, y_val_cv = train_data.iloc[train_idx]['sii'], train_data.iloc[val_idx]['sii']\n\n \n\n# # 删除 'id' 列，防止泄露\n\n# if 'id' in X_train_cv.columns:\n\n# X_train_cv = X_train_cv.drop(columns=['id'])\n\n# if 'id' in X_val_cv.columns:\n\n# X_val_cv = X_val_cv.drop(columns=['id'])\n\n \n\n# train_pool = Pool(X_train_cv, label=y_train_cv)\n\n# val_pool = Pool(X_val_cv, label=y_val_cv)\n\n \n\n# model = CatBoostClassifier(**best_params)\n\n# model.fit(train_pool, eval_set=val_pool, verbose=100)\n\n \n\n# y_val_pred_proba = model.predict_proba(X_val_cv)\n\n# y_val_pred_continuous = np.dot(y_val_pred_proba, class_numbers)\n\n \n\n# oof_non_rounded[val_idx] = y_val_pred_continuous\n\n# y_val_pred_rounded = np.round(y_val_pred_continuous).astype(int)\n\n \n\n# accuracy = accuracy_score(y_val_cv, y_val_pred_rounded)\n\n# qwk = calculate_qwk(y_val_cv, y_val_pred_rounded)\n\n# print(f\"Fold {fold+1} - Accuracy: {accuracy:.4f}, QWK: {qwk:.4f}\")\n\n \n\n# train_S.append(accuracy)\n\n# test_S.append(qwk)\n\n \n\n# X_test_cv = valid_data.drop(columns=['id']) if 'id' in valid_data.columns else valid_data.copy()\n\n# y_test_pred_proba = model.predict_proba(X_test_cv)\n\n# y_test_pred_continuous = np.dot(y_test_pred_proba, class_numbers)\n\n# test_preds[:, fold] = y_test_pred_continuous\n\n\n\n# # 输出平均指标\n\n# print(f\"Mean Train Accuracy --> {np.mean(train_S):.4f}\")\n\n# print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n\n\n# # 优化 QWK 阈值\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n\n# thresholds = np.sort(thresholds)\n\n# oof_tuned = np.digitize(oof_non_rounded, bins=thresholds)\n\n# return -calculate_qwk(y_true, oof_tuned)\n\n\n\n# KappaOptimizer = minimize(\n\n# evaluate_predictions,\n\n# x0=[0.5, 1.5, 2.5],\n\n# args=(train_data['sii'], oof_non_rounded),\n\n# method='Nelder-Mead'\n\n# )\n\n\n\n# optimized_thresholds = np.sort(KappaOptimizer.x)\n\n# oof_tuned = np.digitize(oof_non_rounded, bins=optimized_thresholds)\n\n# tKappa = calculate_qwk(train_data['sii'], oof_tuned)\n\n\n\n# print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n\n\n# # 最终测试集的预测值\n\n# final_test_preds = test_preds.mean(axis=1)\n\n# final_test_preds_rounded = np.digitize(final_test_preds, bins=optimized_thresholds)\n\n\n\n# # 计算 QWK 分数\n\n# qwk_score = calculate_qwk(valid_sii, final_test_preds_rounded)\n\n# print(f\"QWK 分数: {qwk_score:.4f}\")\n\n\n\n# # 输出分类报告\n\n# print(\"\\nClassification Report:\")\n\n# print(classification_report(valid_sii, final_test_preds_rounded))\n\n\n\n# # 输出混淆矩阵\n\n# conf_matrix = confusion_matrix(valid_sii, final_test_preds_rounded)\n\n# print(\"\\nConfusion Matrix:\")\n\n# print(conf_matrix)\n\n\n\n# # 绘制混淆矩阵热力图\n\n# plt.figure(figsize=(10, 7))\n\n# sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n\n# xticklabels=np.unique(valid_sii),\n\n# yticklabels=np.unique(valid_sii))\n\n# plt.xlabel('Predicted Label')\n\n# plt.ylabel('True Label')\n\n# plt.title('Confusion Matrix')\n\n# plt.show()","metadata":{"papermill":{"duration":0.051166,"end_time":"2024-11-20T15:17:17.219748","exception":false,"start_time":"2024-11-20T15:17:17.168582","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.605620Z","iopub.execute_input":"2024-12-12T14:54:39.605898Z","iopub.status.idle":"2024-12-12T14:54:39.619008Z","shell.execute_reply.started":"2024-12-12T14:54:39.605874Z","shell.execute_reply":"2024-12-12T14:54:39.618202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 导入必要的库\nimport numpy as np\nimport pandas as pd\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nfrom scipy.optimize import minimize\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom catboost import CatBoostClassifier\nimport warnings\n\nwarnings.filterwarnings('ignore')\n\n# 设置随机种子和参数\nn_splits = 5\nSEED = 42\nnum_classes = train_data['sii'].nunique()\nclass_numbers = np.arange(num_classes)\n\n# 初始化数组用于存储验证集和测试集的预测\noof_non_rounded = np.zeros(len(train_data), dtype=float)\noof_rounded = np.zeros(len(train_data), dtype=int)\nvalid_preds = np.zeros((len(valid_data), n_splits))\ntest_preds = np.zeros((len(test), n_splits)) # 新增用于存储 test 数据集的预测\n\ntrain_S = []\ntest_S = []\n\n# # 定义并训练模型\n# model = CatBoostClassifier(\n# iterations=1803,\n# learning_rate=0.07678030468321072,\n# depth=5,\n# l2_leaf_reg=0.013832267688479753,\n# loss_function='MultiClass',\n# random_seed=SEED,\n# verbose=100\n# )\n\n# 定义并训练模型\nmodel = CatBoostClassifier(\n iterations=500,\n learning_rate=0.023,\n depth=6,\n loss_function='MultiClass',\n random_seed=SEED,\n verbose=100,\n\n)\n\n# 交叉验证\nSKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\nfor fold, (train_idx, val_idx) in enumerate(tqdm(SKF.split(train_data, train_data['sii']), desc=\"Training Folds\", total=n_splits)):\n print(f\"\\nFold {fold+1}\")\n \n # 划分训练集和验证集\n X_train_cv = train_data.iloc[train_idx].drop(columns=['sii']).copy()\n X_val_cv = train_data.iloc[val_idx].drop(columns=['sii']).copy()\n y_train_cv = train_data.iloc[train_idx]['sii'].copy()\n y_val_cv = train_data.iloc[val_idx]['sii'].copy()\n\n # 删除 'id' 列，防止泄露\n if 'id' in X_train_cv.columns:\n X_train_cv = X_train_cv.drop(columns=['id'])\n if 'id' in X_val_cv.columns:\n X_val_cv = X_val_cv.drop(columns=['id'])\n \n # 训练模型\n model.fit(X_train_cv, y_train_cv, eval_set=(X_val_cv, y_val_cv), verbose=False)\n\n # 验证集上的概率预测\n y_val_pred_proba = model.predict_proba(X_val_cv)\n y_val_pred_continuous = np.dot(y_val_pred_proba, class_numbers)\n \n # 存储验证集预测结果\n oof_non_rounded[val_idx] = y_val_pred_continuous\n y_val_pred_rounded = np.round(y_val_pred_continuous).astype(int)\n oof_rounded[val_idx] = y_val_pred_rounded\n \n # 计算指标\n accuracy = accuracy_score(y_val_cv, y_val_pred_rounded)\n qwk = calculate_qwk(y_val_cv, y_val_pred_rounded)\n print(f\"Fold {fold+1} - Accuracy: {accuracy:.4f}, QWK: {qwk:.4f}\")\n \n train_S.append(accuracy)\n test_S.append(qwk)\n \n # 在验证集上进行预测并存储\n X_valid_cv = valid_data.drop(columns=['id']) if 'id' in valid_data.columns else valid_data.copy()\n y_valid_pred_proba = model.predict_proba(X_valid_cv)\n y_valid_pred_continuous = np.dot(y_valid_pred_proba, class_numbers)\n valid_preds[:, fold] = y_valid_pred_continuous\n\n # 在测试集上进行预测并存储\n X_test_cv = test.drop(columns=['id']) if 'id' in test.columns else test.copy()\n y_test_pred_proba = model.predict_proba(X_test_cv)\n y_test_pred_continuous = np.dot(y_test_pred_proba, class_numbers)\n test_preds[:, fold] = y_test_pred_continuous\n\n# 输出平均指标\nprint(f\"\\nMean Train Accuracy --> {np.mean(train_S):.4f}\")\nprint(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\ntarget_class_proportions = [0.58, 0.26, 0.13, 0.01] # 请根据实际情况修改\n\n# 确保比例和为1\ntarget_class_proportions = np.array(target_class_proportions)\ntarget_class_proportions = target_class_proportions / target_class_proportions.sum()\n\n# 计算累积比例（不包括最后一个类别）\ncumulative_proportions = np.cumsum(target_class_proportions)[:-1]\n\n# 根据累积比例在预测值中找到对应的阈值\ndef adjust_thresholds_to_match_proportions(preds, target_proportions):\n \"\"\"\n 调整阈值以匹配目标类别比例\n\n preds: 连续预测值\n target_proportions: 目标类别比例\n \"\"\"\n # 对预测值排序\n sorted_preds = np.sort(preds)\n # 计算累积目标比例（不包括最后一个）\n cumulative_proportions = np.cumsum(target_proportions)[:-1]\n # 在排序的预测值中找到对应的阈值\n thresholds = np.quantile(sorted_preds, cumulative_proportions)\n return thresholds\n\n# 调整阈值以匹配目标类别比例\noptimized_thresholds = adjust_thresholds_to_match_proportions(oof_non_rounded, target_class_proportions)\n\ndef threshold_rounder(values, thresholds):\n return np.digitize(values, bins=thresholds)\n\n# 使用优化的阈值对训练集预测进行调整\noof_tuned = threshold_rounder(oof_non_rounded, optimized_thresholds)\ntKappa = calculate_qwk(train_data['sii'], oof_tuned)\n\nprint(f\"----> || Optimized QWK SCORE :: {tKappa:.3f}\")\n\n# 最终验证集的预测值\nfinal_valid_preds = valid_preds.mean(axis=1)\nfinal_valid_preds_rounded = threshold_rounder(final_valid_preds, optimized_thresholds)\n\n# 计算验证集的 QWK 分数\nqwk_score = calculate_qwk(valid_sii, final_valid_preds_rounded)\nprint(f\"验证集 QWK 分数: {qwk_score:.4f}\")\n\n# 输出验证集的分类报告\nprint(\"\\nValidation Classification Report:\")\nprint(classification_report(valid_sii, final_valid_preds_rounded))\n\n# 输出验证集的混淆矩阵\nconf_matrix = confusion_matrix(valid_sii, final_valid_preds_rounded)\nprint(\"\\nValidation Confusion Matrix:\")\nprint(conf_matrix)\n\n# 绘制验证集的混淆矩阵热力图\nplt.figure(figsize=(10, 7))\nsns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n xticklabels=np.unique(valid_sii),\n yticklabels=np.unique(valid_sii))\nplt.xlabel('Predicted Label')\nplt.ylabel('True Label')\nplt.title('Validation Confusion Matrix')\nplt.show()\n\n# 最终测试集的预测值\nfinal_test_preds = test_preds.mean(axis=1)\nfinal_test_preds_rounded = threshold_rounder(final_test_preds, optimized_thresholds)\n\n# 将预测结果与 'id' 对应，生成 submission DataFrame\nsubmission1 = pd.DataFrame({\n 'id': test['id'],\n 'sii': final_test_preds_rounded\n})\n\n# 显示提交文件的前几行\nprint(\"\\nSubmission1 Preview:\")\nprint(submission1.head())\n\n# 如果需要，将提交文件保存为 CSV\nsubmission1.to_csv('submission.csv', index=False)\n\nsubmission1","metadata":{"papermill":{"duration":83.448902,"end_time":"2024-11-20T15:18:40.707426","exception":false,"start_time":"2024-11-20T15:17:17.258524","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:54:39.620298Z","iopub.execute_input":"2024-12-12T14:54:39.620579Z","iopub.status.idle":"2024-12-12T14:55:22.383738Z","shell.execute_reply.started":"2024-12-12T14:54:39.620552Z","shell.execute_reply":"2024-12-12T14:55:22.382805Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"对withwime的训练","metadata":{"papermill":{"duration":0.03966,"end_time":"2024-11-20T15:18:40.788383","exception":false,"start_time":"2024-11-20T15:18:40.748723","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from sklearn.calibration import CalibratedClassifierCV # 使用CalibratedClassifierCV进行优化 \n\n# from imblearn.over_sampling import RandomOverSampler\n\n# from imblearn.over_sampling import SMOTE\n\n# from imblearn.over_sampling import KMeansSMOTE\n\n\n\n\n\n# # 设定参数\n\n\n\n\n\n# n_splits = 5\n\n# SEED = 42\n\n# num_classes = train_data_withtime['sii'].nunique()\n\n# class_numbers = np.arange(num_classes)\n\n\n\n# # 初始化数组用于存储验证集和测试集的预测\n\n# oof_non_rounded = np.zeros(len(train_data_withtime), dtype=float)\n\n# oof_rounded = np.zeros(len(train_data_withtime), dtype=int)\n\n# test_preds = np.zeros((len(valid_data_withtime), n_splits))\n\n\n\n# train_S = []\n\n# test_S = []\n\n\n\n# # 定义并训练模型\n\n# model = CatBoostClassifier(\n\n# iterations=1000,\n\n# learning_rate=0.05,\n\n# depth=6,\n\n# loss_function='MultiClass',\n\n# random_seed=SEED,\n\n# verbose=100\n\n# )\n\n\n\n# # model = CatBoostClassifier(\n\n# # iterations=1099,\n\n# # learning_rate=0.0137,\n\n# # depth=6,\n\n# # l2_leaf_reg=0.003543,\n\n# # loss_function='MultiClass',\n\n# # random_seed=SEED,\n\n# # verbose=100\n\n# # )\n\n\n\n\n\n# # 定义并训练 XGBoost 模型\n\n# # model = XGBClassifier(\n\n# # n_estimators=1000,\n\n# # learning_rate=0.05,\n\n# # max_depth=6,\n\n# # objective='multi:softprob',\n\n# # random_state=SEED,\n\n# # eval_metric='mlogloss',\n\n# # use_label_encoder=False\n\n# # )\n\n\n\n# # 交叉验证\n\n# SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n\n\n# for fold, (train_idx, val_idx) in enumerate(tqdm(SKF.split(train_data_withtime, train_data_withtime['sii']), desc=\"Training Folds\", total=n_splits)):\n\n# print(f\"Fold {fold+1}\")\n\n \n\n# # 划分训练集和验证集\n\n# X_train_cv, X_val_cv = train_data_withtime.iloc[train_idx].drop(columns=['sii']), train_data_withtime.iloc[val_idx].drop(columns=['sii'])\n\n# y_train_cv, y_val_cv = train_data_withtime.iloc[train_idx]['sii'], train_data_withtime.iloc[val_idx]['sii']\n\n\n\n\n\n# # # oversampler = KMeansSMOTE(random_state=42, cluster_balance_threshold=0.1)\n\n# # oversampler = RandomOverSampler(random_state=42)\n\n# # # oversampler = SMOTE(random_state=SEED)\n\n# # X_train_resampled, y_train_resampled = oversampler.fit_resample(X_train_cv, y_train_cv)\n\n \n\n# # # 将过采样后的数据组合成新的 DataFrame（只用于当前折叠）\n\n# # train_data_resampled = pd.DataFrame(X_train_resampled, columns=X_train_cv.columns)\n\n# # train_data_resampled['sii'] = y_train_resampled\n\n\n\n# # 删除 'id' 列，防止泄露\n\n# if 'id' in X_train_cv.columns:\n\n# X_train_cv = X_train_cv.drop(columns=['id'])\n\n# if 'id' in X_val_cv.columns:\n\n# X_val_cv = X_val_cv.drop(columns=['id'])\n\n \n\n# model.fit(X_train_cv, y_train_cv, eval_set=(X_val_cv, y_val_cv), verbose=False)\n\n# # model.fit(X_train_cv, y_train_cv, verbose=False)\n\n\n\n\n\n# # 验证集上的概率预测\n\n# y_val_pred_proba = model.predict_proba(X_val_cv)\n\n# y_val_pred_continuous = np.dot(y_val_pred_proba, class_numbers)\n\n \n\n# # 存储预测结果\n\n# oof_non_rounded[val_idx] = y_val_pred_continuous\n\n# y_val_pred_rounded = np.round(y_val_pred_continuous).astype(int)\n\n# oof_rounded[val_idx] = y_val_pred_rounded\n\n \n\n# # 计算指标\n\n# accuracy = accuracy_score(y_val_cv, y_val_pred_rounded)\n\n# qwk = calculate_qwk(y_val_cv, y_val_pred_rounded)\n\n# print(f\"Fold {fold+1} - Accuracy: {accuracy:.4f}, QWK: {qwk:.4f}\")\n\n \n\n# train_S.append(accuracy)\n\n# test_S.append(qwk)\n\n \n\n# # 在测试集上进行预测\n\n# X_test_cv = valid_data_withtime.drop(columns=['id']) if 'id' in valid_data_withtime.columns else valid_data_withtime.copy()\n\n# y_test_pred_proba = model.predict_proba(X_test_cv)\n\n# y_test_pred_continuous = np.dot(y_test_pred_proba, class_numbers)\n\n# test_preds[:, fold] = y_test_pred_continuous\n\n\n\n# # 输出平均指标\n\n# print(f\"Mean Train Accuracy --> {np.mean(train_S):.4f}\")\n\n# print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n\n\n# # 优化 QWK 阈值\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n\n# # 确保 thresholds 单调递增\n\n# thresholds = np.sort(thresholds)\n\n# oof_tuned = np.digitize(oof_non_rounded, bins=thresholds)\n\n# return -calculate_qwk(y_true, oof_tuned)\n\n\n\n# KappaOptimizer = minimize(\n\n# evaluate_predictions,\n\n# x0=[0.5, 1.5, 2.5],\n\n# args=(train_data_withtime['sii'], oof_non_rounded),\n\n# method='Nelder-Mead'\n\n# )\n\n\n\n# assert KappaOptimizer.success, \"Optimization did not converge.\"\n\n\n\n# # 使用优化后的阈值，确保单调递增\n\n# optimized_thresholds = np.sort(KappaOptimizer.x)\n\n\n\n# def threshold_rounder(values, thresholds):\n\n# return np.digitize(values, bins=thresholds)\n\n\n\n# # 使用优化的阈值对训练集预测进行调整\n\n# oof_tuned = threshold_rounder(oof_non_rounded, optimized_thresholds)\n\n# tKappa = calculate_qwk(train_data_withtime['sii'], oof_tuned)\n\n\n\n# print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n\n\n# # 最终测试集的预测值\n\n# final_test_preds = test_preds.mean(axis=1)\n\n# final_test_preds_rounded = threshold_rounder(final_test_preds, optimized_thresholds)\n\n\n\n# # 计算 QWK 分数\n\n# qwk_score = calculate_qwk(valid_sii_withtime, final_test_preds_rounded)\n\n# print(f\"QWK 分数: {qwk_score:.4f}\")\n\n\n\n# # 输出分类报告\n\n# print(\"\\nClassification Report:\")\n\n# print(classification_report(valid_sii_withtime, final_test_preds_rounded))\n\n\n\n# # 输出混淆矩阵\n\n\n\n# conf_matrix = confusion_matrix(valid_sii_withtime, final_test_preds_rounded)\n\n# print(\"\\nConfusion Matrix:\")\n\n# print(conf_matrix)\n\n\n\n# # 绘制混淆矩阵热力图\n\n# plt.figure(figsize=(10, 7))\n\n# sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n\n# xticklabels=np.unique(valid_sii_withtime),\n\n# yticklabels=np.unique(valid_sii_withtime))\n\n# plt.xlabel('Predicted Label')\n\n# plt.ylabel('True Label')\n\n# plt.title('Confusion Matrix')\n\n# plt.show()","metadata":{"papermill":{"duration":0.052723,"end_time":"2024-11-20T15:18:40.881646","exception":false,"start_time":"2024-11-20T15:18:40.828923","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:22.385452Z","iopub.execute_input":"2024-12-12T14:55:22.385824Z","iopub.status.idle":"2024-12-12T14:55:22.396692Z","shell.execute_reply.started":"2024-12-12T14:55:22.385785Z","shell.execute_reply":"2024-12-12T14:55:22.395753Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"生成最后的提交","metadata":{"papermill":{"duration":0.039958,"end_time":"2024-11-20T15:18:40.961845","exception":false,"start_time":"2024-11-20T15:18:40.921887","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# # 导入必要的库\n\n# import numpy as np\n\n# import pandas as pd\n\n# from sklearn.calibration import CalibratedClassifierCV\n\n# from sklearn.model_selection import StratifiedKFold\n\n# from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n\n# from scipy.optimize import minimize\n\n# from tqdm import tqdm\n\n# import matplotlib.pyplot as plt\n\n# import seaborn as sns\n\n# from catboost import CatBoostClassifier\n\n# import warnings\n\n\n\n# warnings.filterwarnings('ignore')\n\n\n\n# # 设置随机种子和参数\n\n# n_splits = 5\n\n# SEED = 42\n\n# num_classes = train_data['sii'].nunique()\n\n# class_numbers = np.arange(num_classes)\n\n\n\n# # 初始化数组用于存储验证集和测试集的预测\n\n# oof_non_rounded = np.zeros(len(train_data), dtype=float)\n\n# oof_rounded = np.zeros(len(train_data), dtype=int)\n\n# valid_preds = np.zeros((len(valid_data), n_splits))\n\n# test_preds = np.zeros((len(test), n_splits)) # 新增用于存储 test 数据集的预测\n\n\n\n# train_S = []\n\n# test_S = []\n\n\n\n# # 定义并训练模型\n\n# model = CatBoostClassifier(\n# iterations=1000,\n# learning_rate=0.05,\n# depth=6,\n# loss_function='MultiClass',\n# random_seed=SEED,\n# verbose=100\n# )\n\n\n\n# # 交叉验证\n\n# SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n\n\n# for fold, (train_idx, val_idx) in enumerate(tqdm(SKF.split(train_data, train_data['sii']), desc=\"Training Folds\", total=n_splits)):\n\n# print(f\"\\nFold {fold+1}\")\n\n \n\n# # 划分训练集和验证集\n\n# X_train_cv = train_data.iloc[train_idx].drop(columns=['sii']).copy()\n\n# X_val_cv = train_data.iloc[val_idx].drop(columns=['sii']).copy()\n\n# y_train_cv = train_data.iloc[train_idx]['sii'].copy()\n\n# y_val_cv = train_data.iloc[val_idx]['sii'].copy()\n\n\n\n# # 删除 'id' 列，防止泄露\n\n# if 'id' in X_train_cv.columns:\n\n# X_train_cv = X_train_cv.drop(columns=['id'])\n\n# if 'id' in X_val_cv.columns:\n\n# X_val_cv = X_val_cv.drop(columns=['id'])\n\n \n\n# # 训练模型\n\n# model.fit(X_train_cv, y_train_cv, eval_set=(X_val_cv, y_val_cv), verbose=False)\n\n\n\n# # 验证集上的概率预测\n\n# y_val_pred_proba = model.predict_proba(X_val_cv)\n\n# y_val_pred_continuous = np.dot(y_val_pred_proba, class_numbers)\n\n \n\n# # 存储验证集预测结果\n\n# oof_non_rounded[val_idx] = y_val_pred_continuous\n\n# y_val_pred_rounded = np.round(y_val_pred_continuous).astype(int)\n\n# oof_rounded[val_idx] = y_val_pred_rounded\n\n \n\n# # 计算指标\n\n# accuracy = accuracy_score(y_val_cv, y_val_pred_rounded)\n\n# qwk = calculate_qwk(y_val_cv, y_val_pred_rounded)\n\n# print(f\"Fold {fold+1} - Accuracy: {accuracy:.4f}, QWK: {qwk:.4f}\")\n\n \n\n# train_S.append(accuracy)\n\n# test_S.append(qwk)\n\n \n\n# # 在验证集上进行预测并存储\n\n# X_valid_cv = valid_data.drop(columns=['id']) if 'id' in valid_data.columns else valid_data.copy()\n\n# y_valid_pred_proba = model.predict_proba(X_valid_cv)\n\n# y_valid_pred_continuous = np.dot(y_valid_pred_proba, class_numbers)\n\n# valid_preds[:, fold] = y_valid_pred_continuous\n\n\n\n# # 在测试集上进行预测并存储\n\n# X_test_cv = test.drop(columns=['id']) if 'id' in test.columns else test.copy()\n\n# y_test_pred_proba = model.predict_proba(X_test_cv)\n\n# y_test_pred_continuous = np.dot(y_test_pred_proba, class_numbers)\n\n# test_preds[:, fold] = y_test_pred_continuous\n\n\n\n# # 输出平均指标\n\n# print(f\"\\nMean Train Accuracy --> {np.mean(train_S):.4f}\")\n\n# print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n\n\n# # 优化 QWK 阈值\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n\n# # 确保 thresholds 单调递增\n\n# thresholds = np.sort(thresholds)\n\n# oof_tuned = np.digitize(oof_non_rounded, bins=thresholds)\n\n# return -calculate_qwk(y_true, oof_tuned)\n\n\n\n# KappaOptimizer = minimize(\n\n# evaluate_predictions,\n\n# x0=[0.5, 1.5, 2.5],\n\n# args=(train_data['sii'], oof_non_rounded),\n\n# method='Nelder-Mead'\n\n# )\n\n\n\n# assert KappaOptimizer.success, \"Optimization did not converge.\"\n\n\n\n# # 使用优化后的阈值，确保单调递增\n\n# optimized_thresholds = np.sort(KappaOptimizer.x)\n\n\n\n# def threshold_rounder(values, thresholds):\n\n# return np.digitize(values, bins=thresholds)\n\n\n\n# # 使用优化的阈值对训练集预测进行调整\n\n# oof_tuned = threshold_rounder(oof_non_rounded, optimized_thresholds)\n\n# tKappa = calculate_qwk(train_data['sii'], oof_tuned)\n\n\n\n# print(f\"----> || Optimized QWK SCORE :: {tKappa:.3f}\")\n\n\n\n# # 最终验证集的预测值\n\n# final_valid_preds = valid_preds.mean(axis=1)\n\n# final_valid_preds_rounded = threshold_rounder(final_valid_preds, optimized_thresholds)\n\n\n\n# # 计算验证集的 QWK 分数\n\n# qwk_score = calculate_qwk(valid_sii, final_valid_preds_rounded)\n\n# print(f\"验证集 QWK 分数: {qwk_score:.4f}\")\n\n\n\n# # 输出验证集的分类报告\n\n# print(\"\\nValidation Classification Report:\")\n\n# print(classification_report(valid_sii, final_valid_preds_rounded))\n\n\n\n# # 输出验证集的混淆矩阵\n\n# conf_matrix = confusion_matrix(valid_sii, final_valid_preds_rounded)\n\n# print(\"\\nValidation Confusion Matrix:\")\n\n# print(conf_matrix)\n\n\n\n# # 绘制验证集的混淆矩阵热力图\n\n# plt.figure(figsize=(10, 7))\n\n# sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n\n# xticklabels=np.unique(valid_sii),\n\n# yticklabels=np.unique(valid_sii))\n\n# plt.xlabel('Predicted Label')\n\n# plt.ylabel('True Label')\n\n# plt.title('Validation Confusion Matrix')\n\n# plt.show()\n\n\n\n# # 最终测试集的预测值\n\n# final_test_preds = test_preds.mean(axis=1)\n\n# final_test_preds_rounded = threshold_rounder(final_test_preds, optimized_thresholds)\n\n\n\n# # 将预测结果与 'id' 对应，生成 submission DataFrame\n\n# submission = pd.DataFrame({\n\n# 'id': test['id'],\n\n# 'sii': final_test_preds_rounded\n\n# })\n\n\n\n# # 显示提交文件的前几行\n\n# print(\"\\nSubmission Preview:\")\n\n# print(submission.head())\n\n\n\n# # 如果需要，将提交文件保存为 CSV\n\n# submission.to_csv('submission.csv', index=False)\n","metadata":{"papermill":{"duration":0.052235,"end_time":"2024-11-20T15:18:41.056291","exception":false,"start_time":"2024-11-20T15:18:41.004056","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:22.398272Z","iopub.execute_input":"2024-12-12T14:55:22.398683Z","iopub.status.idle":"2024-12-12T14:55:22.414293Z","shell.execute_reply.started":"2024-12-12T14:55:22.398639Z","shell.execute_reply":"2024-12-12T14:55:22.413210Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"使用CalibratedClassifierCV 和 TunedThresholdClassifierCV进行训练","metadata":{"papermill":{"duration":0.039481,"end_time":"2024-11-20T15:18:41.135921","exception":false,"start_time":"2024-11-20T15:18:41.096440","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# from catboost import CatBoostClassifier\n\n# from sklearn.metrics import accuracy_score\n\n# from sklearn.model_selection import StratifiedKFold\n\n# from sklearn.base import BaseEstimator, ClassifierMixin\n\n# from sklearn.calibration import CalibratedClassifierCV\n\n# from tqdm import tqdm\n\n# from scipy.optimize import minimize\n\n# from sklearn.metrics import cohen_kappa_score as calculate_qwk\n\n\n\n# # 定义 TunedThresholdClassifierCV\n\n# class TunedThresholdClassifierCV(BaseEstimator, ClassifierMixin):\n\n# def __init__(self, base_model, n_classes, n_splits=5, seed=42):\n\n# self.base_model = base_model\n\n# self.n_classes = n_classes\n\n# self.n_splits = n_splits\n\n# self.seed = seed\n\n# self.thresholds_ = None\n\n\n\n# def fit(self, X, y):\n\n# skf = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=self.seed)\n\n# oof_preds = np.zeros((len(X), self.n_classes))\n\n# self.models_ = []\n\n\n\n# # Train each fold with CalibratedClassifierCV\n\n# for train_idx, val_idx in skf.split(X, y):\n\n# X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n\n# y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n\n\n# # Train base model and apply CalibratedClassifierCV\n\n# model = self.base_model\n\n# model.fit(X_train, y_train, eval_set=(X_val, y_val), verbose=False)\n\n# calibrated_model = CalibratedClassifierCV(model, method='isotonic', cv='prefit')\n\n# calibrated_model.fit(X_val, y_val)\n\n# self.models_.append(calibrated_model)\n\n\n\n# # Get out-of-fold predictions\n\n# oof_preds[val_idx] = calibrated_model.predict_proba(X_val)\n\n\n\n# # Optimize thresholds\n\n# self.thresholds_ = self.optimize_thresholds(oof_preds, y)\n\n\n\n# def optimize_thresholds(self, oof_preds, y_true):\n\n# def qwk_loss(thresholds):\n\n# preds = np.digitize(np.dot(oof_preds, np.arange(self.n_classes)), bins=thresholds)\n\n# return -calculate_qwk(y_true, preds)\n\n\n\n# initial_thresholds = np.linspace(0.5, self.n_classes - 1.5, self.n_classes - 1)\n\n# result = minimize(qwk_loss, initial_thresholds, method='Nelder-Mead')\n\n# return np.sort(result.x)\n\n\n\n# def predict(self, X):\n\n# preds_proba = np.mean([model.predict_proba(X) for model in self.models_], axis=0)\n\n# preds_continuous = np.dot(preds_proba, np.arange(self.n_classes))\n\n# return np.digitize(preds_continuous, bins=self.thresholds_)\n\n\n\n# def predict_proba(self, X):\n\n# return np.mean([model.predict_proba(X) for model in self.models_], axis=0)\n\n\n\n# # Initialize parameters\n\n# n_splits = 5\n\n# SEED = 42\n\n# num_classes = train_data['sii'].nunique()\n\n# class_numbers = np.arange(num_classes)\n\n\n\n# # Define the CatBoost model\n\n# base_model = CatBoostClassifier(\n\n# iterations=1000,\n\n# learning_rate=0.05,\n\n# depth=6,\n\n# loss_function='MultiClass',\n\n# random_seed=SEED,\n\n# verbose=100\n\n# )\n\n\n\n# # Initialize TunedThresholdClassifierCV with CalibratedClassifierCV applied\n\n# tuned_model = TunedThresholdClassifierCV(base_model=base_model, n_classes=num_classes, n_splits=n_splits, seed=SEED)\n\n\n\n# # Prepare data\n\n# X_train, y_train = train_data.drop(columns=['sii']), train_data['sii']\n\n# if 'id' in X_train.columns:\n\n# X_train = X_train.drop(columns=['id'])\n\n\n\n# # Fit the model\n\n# tuned_model.fit(X_train, y_train)\n\n\n\n# # Out-of-fold predictions\n\n# oof_rounded = tuned_model.predict(X_train)\n\n# oof_non_rounded = np.dot(tuned_model.predict_proba(X_train), class_numbers)\n\n# qwk_oof = calculate_qwk(y_train, oof_rounded)\n\n# print(f\"Optimized QWK on Training Set: {qwk_oof:.4f}\")\n\n\n\n# # Final test set predictions\n\n# X_test = valid_data.drop(columns=['id']) if 'id' in valid_data.columns else valid_data.copy()\n\n# final_test_preds_rounded = tuned_model.predict(X_test)\n\n# final_test_preds_non_rounded = np.dot(tuned_model.predict_proba(X_test), class_numbers)\n\n\n\n# qwk_score = calculate_qwk(valid_sii, final_test_preds_rounded)\n\n# print(final_test_preds_rounded)\n\n# print(valid_sii)\n\n# print(f\"Final QWK Score on Validation Set: {qwk_score:.4f}\")\n\n\n\n# # 输出分类报告\n\n# print(\"\\nClassification Report:\")\n\n# print(classification_report(valid_sii, final_test_preds_rounded))\n\n\n\n# # 输出混淆矩阵\n\n# conf_matrix = confusion_matrix(valid_sii, final_test_preds_rounded)\n\n# print(\"\\nConfusion Matrix:\")\n\n# print(conf_matrix)\n\n\n\n# # 绘制混淆矩阵热力图\n\n# plt.figure(figsize=(10, 7))\n\n# sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n\n# xticklabels=np.unique(valid_sii),\n\n# yticklabels=np.unique(valid_sii))\n\n# plt.xlabel('Predicted Label')\n\n# plt.ylabel('True Label')\n\n# plt.title('Confusion Matrix')\n\n# plt.show()","metadata":{"papermill":{"duration":0.05,"end_time":"2024-11-20T15:18:41.225517","exception":false,"start_time":"2024-11-20T15:18:41.175517","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:22.415834Z","iopub.execute_input":"2024-12-12T14:55:22.416214Z","iopub.status.idle":"2024-12-12T14:55:22.430879Z","shell.execute_reply.started":"2024-12-12T14:55:22.416174Z","shell.execute_reply":"2024-12-12T14:55:22.430218Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"使用NGBOOST进行训练","metadata":{"papermill":{"duration":0.039875,"end_time":"2024-11-20T15:18:41.305717","exception":false,"start_time":"2024-11-20T15:18:41.265842","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# # 导入必要的库\n\n# from sklearn.calibration import CalibratedClassifierCV\n\n# from ngboost import NGBClassifier\n\n# from ngboost.distns import k_categorical\n\n# from sklearn.model_selection import StratifiedKFold\n\n# from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n\n# from scipy.optimize import minimize\n\n# from tqdm import tqdm\n\n# import numpy as np\n\n# import pandas as pd\n\n# import matplotlib.pyplot as plt\n\n# import seaborn as sns\n\n\n\n# # 如果需要计算 QWK（Quadratic Weighted Kappa），请确保定义了 calculate_qwk 函数\n\n# # from sklearn.metrics import cohen_kappa_score\n\n# def calculate_qwk(y_true, y_pred):\n\n# # 您可以使用合适的 QWK 实现，这里假设您已有实现\n\n# # 例如，使用 cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n# from sklearn.metrics import cohen_kappa_score\n\n# return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n\n\n# # 设定参数\n\n# n_splits = 5\n\n# SEED = 42\n\n# num_classes = train_data['sii'].nunique()\n\n# class_numbers = np.arange(num_classes)\n\n\n\n# # 初始化数组用于存储验证集和测试集的预测\n\n# oof_non_rounded = np.zeros(len(train_data), dtype=float)\n\n# oof_rounded = np.zeros(len(train_data), dtype=int)\n\n# test_preds = np.zeros((len(valid_data), n_splits))\n\n\n\n# train_S = []\n\n# test_S = []\n\n\n\n# # 定义并初始化 NGBoost 模型\n\n# from ngboost import NGBClassifier\n\n# from ngboost.distns import k_categorical\n\n\n\n# model = NGBClassifier(\n\n# Dist=k_categorical(num_classes),\n\n# n_estimators=1000,\n\n# learning_rate=0.05,\n\n# verbose=True,\n\n# random_state=SEED\n\n# )\n\n\n\n# # 1. 提取 'sii' 列\n\n# sii_column = train_data['sii'].values\n\n# train_features = train_data.drop(columns=['sii']).copy()\n\n\n\n# # 2. 确保 'sii' 列不在 valid_data 中\n\n# valid_features = valid_data.copy()\n\n\n\n# # 3. 合并 train_features 和 valid_features\n\n# combined_data = pd.concat([train_features, valid_features], ignore_index=True)\n\n\n\n# # 4. 将无穷大和极端数值替换为 NaN\n\n# combined_data.replace([np.inf, -np.inf], np.nan, inplace=True)\n\n\n\n# # 5. 识别数值型和非数值型列\n\n# numeric_cols = combined_data.select_dtypes(include=['number']).columns\n\n# categorical_cols = combined_data.select_dtypes(exclude=['number']).columns\n\n\n\n# # 6. 对数值型列进行缺失值填充（中位数）\n\n# if len(numeric_cols) > 0:\n\n# numeric_imputer = SimpleImputer(strategy='median')\n\n# combined_data[numeric_cols] = numeric_imputer.fit_transform(combined_data[numeric_cols])\n\n\n\n# # 7. 对非数值型列进行缺失值填充（众数）\n\n# if len(categorical_cols) > 0:\n\n# categorical_imputer = SimpleImputer(strategy='most_frequent')\n\n# combined_data[categorical_cols] = categorical_imputer.fit_transform(combined_data[categorical_cols])\n\n\n\n# # 8. 将数据拆分回 train_data 和 valid_data\n\n# train_filled_df = combined_data.iloc[:len(train_features)].reset_index(drop=True)\n\n# valid_filled_df = combined_data.iloc[len(train_features):].reset_index(drop=True)\n\n\n\n# # 9. 将 'sii' 列添加回 train_filled_df\n\n# train_filled_df['sii'] = sii_column\n\n\n\n# # 10. 更新 train_data 和 valid_data\n\n# train_data = train_filled_df\n\n# valid_data = valid_filled_df\n\n\n\n# print(\"train_data 和 valid_data 缺失值填补完毕\")\n\n# # 交叉验证\n\n# SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n\n\n# for fold, (train_idx, val_idx) in enumerate(tqdm(SKF.split(train_data, train_data['sii']), desc=\"Training Folds\", total=n_splits)):\n\n# print(f\"\\nFold {fold+1}\")\n\n \n\n# # 划分训练集和验证集\n\n# X_train_cv = train_data.iloc[train_idx].drop(columns=['sii']).copy()\n\n# X_val_cv = train_data.iloc[val_idx].drop(columns=['sii']).copy()\n\n# y_train_cv = train_data.iloc[train_idx]['sii'].values\n\n# y_val_cv = train_data.iloc[val_idx]['sii'].values\n\n# y_train_cv = y_train_cv.astype(int)\n\n# y_val_cv = y_val_cv.astype(int)\n\n# # 删除 'id' 列，防止泄露\n\n# if 'id' in X_train_cv.columns:\n\n# X_train_cv = X_train_cv.drop(columns=['id'])\n\n# if 'id' in X_val_cv.columns:\n\n# X_val_cv = X_val_cv.drop(columns=['id'])\n\n \n\n# # 训练 NGBoost 模型\n\n# model.fit(X_train_cv, y_train_cv)\n\n \n\n# # 验证集上的概率预测\n\n# y_val_pred_proba = model.predict_proba(X_val_cv)\n\n# y_val_pred_continuous = np.dot(y_val_pred_proba, class_numbers)\n\n \n\n# # 存储预测结果\n\n# oof_non_rounded[val_idx] = y_val_pred_continuous\n\n# y_val_pred_rounded = np.argmax(y_val_pred_proba, axis=1)\n\n# oof_rounded[val_idx] = y_val_pred_rounded\n\n \n\n# # 计算指标\n\n# accuracy = accuracy_score(y_val_cv, y_val_pred_rounded)\n\n# qwk = calculate_qwk(y_val_cv, y_val_pred_rounded)\n\n# print(f\"Fold {fold+1} - Accuracy: {accuracy:.4f}, QWK: {qwk:.4f}\")\n\n \n\n# train_S.append(accuracy)\n\n# test_S.append(qwk)\n\n \n\n# # 在测试集上进行预测\n\n# X_test_cv = valid_data.drop(columns=['id'], errors='ignore').copy()\n\n# y_test_pred_proba = model.predict_proba(X_test_cv)\n\n# y_test_pred_continuous = np.dot(y_test_pred_proba, class_numbers)\n\n# test_preds[:, fold] = y_test_pred_continuous\n\n\n\n# # 输出平均指标\n\n# print(f\"\\nMean Train Accuracy --> {np.mean(train_S):.4f}\")\n\n# print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n\n\n# # 优化 QWK 阈值\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n\n# # 确保 thresholds 单调递增\n\n# thresholds = np.sort(thresholds)\n\n# oof_tuned = np.digitize(oof_non_rounded, bins=thresholds)\n\n# return -calculate_qwk(y_true, oof_tuned)\n\n\n\n# initial_thresholds = np.linspace(0.5, num_classes - 1.5, num=num_classes - 1)\n\n\n\n# KappaOptimizer = minimize(\n\n# evaluate_predictions,\n\n# x0=initial_thresholds,\n\n# args=(train_data['sii'], oof_non_rounded),\n\n# method='Nelder-Mead'\n\n# )\n\n\n\n# assert KappaOptimizer.success, \"Optimization did not converge.\"\n\n\n\n# # 使用优化后的阈值，确保单调递增\n\n# optimized_thresholds = np.sort(KappaOptimizer.x)\n\n\n\n# def threshold_rounder(values, thresholds):\n\n# return np.digitize(values, bins=thresholds)\n\n\n\n# # 使用优化的阈值对训练集预测进行调整\n\n# oof_tuned = threshold_rounder(oof_non_rounded, optimized_thresholds)\n\n# tKappa = calculate_qwk(train_data['sii'], oof_tuned)\n\n\n\n# print(f\"----> || Optimized QWK SCORE :: {tKappa:.3f}\")\n\n\n\n# # 最终测试集的预测值\n\n# final_test_preds = test_preds.mean(axis=1)\n\n# final_test_preds_rounded = threshold_rounder(final_test_preds, optimized_thresholds)\n\n\n\n# # 计算 QWK 分数\n\n# qwk_score = calculate_qwk(valid_sii, final_test_preds_rounded)\n\n# print(f\"QWK 分数: {qwk_score:.4f}\")\n\n\n\n# # 输出分类报告\n\n# print(\"\\nClassification Report:\")\n\n# print(classification_report(valid_sii, final_test_preds_rounded))\n\n\n\n# # 输出混淆矩阵\n\n# conf_matrix = confusion_matrix(valid_sii, final_test_preds_rounded)\n\n# print(\"\\nConfusion Matrix:\")\n\n# print(conf_matrix)\n\n\n\n# # 绘制混淆矩阵热力图\n\n# plt.figure(figsize=(10, 7))\n\n# sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n\n# xticklabels=np.unique(valid_sii),\n\n# yticklabels=np.unique(valid_sii))\n\n# plt.xlabel('Predicted Label')\n\n# plt.ylabel('True Label')\n\n# plt.title('Confusion Matrix')\n\n# plt.show()\n","metadata":{"papermill":{"duration":0.052289,"end_time":"2024-11-20T15:18:41.397586","exception":false,"start_time":"2024-11-20T15:18:41.345297","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:22.432321Z","iopub.execute_input":"2024-12-12T14:55:22.432562Z","iopub.status.idle":"2024-12-12T14:55:22.447012Z","shell.execute_reply.started":"2024-12-12T14:55:22.432539Z","shell.execute_reply":"2024-12-12T14:55:22.446067Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"使用TabPFN进行训练","metadata":{"papermill":{"duration":0.040072,"end_time":"2024-11-20T15:18:41.478340","exception":false,"start_time":"2024-11-20T15:18:41.438268","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# import numpy as np\n\n# import pandas as pd\n\n# from sklearn.metrics import accuracy_score, cohen_kappa_score as calculate_qwk\n\n# from sklearn.model_selection import StratifiedKFold\n\n# from tqdm import tqdm\n\n# from tabpfn import TabPFNClassifier\n\n\n\n# # 设定参数\n\n# n_splits = 5\n\n# SEED = 42\n\n# num_classes = train_data['sii'].nunique()\n\n# class_numbers = np.arange(num_classes)\n\n\n\n# # 初始化数组用于存储验证集和测试集的预测\n\n# oof_preds = np.zeros(len(train_data), dtype=int)\n\n# test_preds = np.zeros((len(valid_data), n_splits), dtype=int)\n\n\n\n# train_S = []\n\n# test_S = []\n\n\n\n# # 交叉验证\n\n# SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n\n\n# for fold, (train_idx, val_idx) in enumerate(tqdm(SKF.split(train_data, train_data['sii']), desc=\"Training Folds\", total=n_splits)):\n\n# print(f\"\\nFold {fold+1}\")\n\n \n\n# # 划分训练集和验证集\n\n# X_train_cv = train_data.iloc[train_idx].drop(columns=['sii'])\n\n# y_train_cv = train_data.iloc[train_idx]['sii']\n\n# X_val_cv = train_data.iloc[val_idx].drop(columns=['sii'])\n\n# y_val_cv = train_data.iloc[val_idx]['sii']\n\n \n\n# # 删除 'id' 列，防止泄露\n\n# if 'id' in X_train_cv.columns:\n\n# X_train_cv = X_train_cv.drop(columns=['id'])\n\n# if 'id' in X_val_cv.columns:\n\n# X_val_cv = X_val_cv.drop(columns=['id'])\n\n \n\n# # 定义并训练 TabPFN 模型\n\n# model = TabPFNClassifier(device='cpu', seed=SEED, N_ensemble_configurations = 32)\n\n# model.fit(X_train_cv.values, y_train_cv.values,overwrite_warning=True)\n\n \n\n# # 验证集上的预测\n\n# y_val_pred = model.predict(X_val_cv.values)\n\n \n\n# # 存储预测结果\n\n# oof_preds[val_idx] = y_val_pred\n\n \n\n# # 计算指标\n\n# accuracy = accuracy_score(y_val_cv, y_val_pred)\n\n# qwk = calculate_qwk(y_val_cv, y_val_pred)\n\n# print(f\"Fold {fold+1} - Accuracy: {accuracy:.4f}, QWK: {qwk:.4f}\")\n\n \n\n# train_S.append(accuracy)\n\n# test_S.append(qwk)\n\n \n\n# # 在测试集上进行预测\n\n# X_test_cv = valid_data.drop(columns=['id']) if 'id' in valid_data.columns else valid_data.copy()\n\n# y_test_pred = model.predict(X_test_cv.values)\n\n# test_preds[:, fold] = y_test_pred\n\n\n\n# # 输出平均指标\n\n# print(f\"\\nMean Train Accuracy --> {np.mean(train_S):.4f}\")\n\n# print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n\n\n# # 最终测试集的预测值（取交叉验证的多数投票）\n\n# from scipy.stats import mode\n\n# final_test_preds = mode(test_preds, axis=1)[0].flatten()\n\n\n\n\n\n\n\n# qwk_score = calculate_qwk(valid_sii, final_test_preds)\n\n# print(f\"\\nFinal QWK Score on Validation Set: {qwk_score:.4f}\")\n\n\n\n\n\n# # 输出分类报告\n\n# print(\"\\nClassification Report:\")\n\n# print(classification_report(valid_sii, final_test_preds))\n\n\n\n# # 输出混淆矩阵\n\n# conf_matrix = confusion_matrix(valid_sii, final_test_preds)\n\n# print(\"\\nConfusion Matrix:\")\n\n# print(conf_matrix)\n\n\n\n# # 绘制混淆矩阵热力图\n\n# plt.figure(figsize=(10, 7))\n\n# sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n\n# xticklabels=np.unique(valid_sii),\n\n# yticklabels=np.unique(valid_sii))\n\n# plt.xlabel('Predicted Label')\n\n# plt.ylabel('True Label')\n\n# plt.title('Confusion Matrix')\n\n# plt.show()\n","metadata":{"papermill":{"duration":0.050161,"end_time":"2024-11-20T15:18:41.568834","exception":false,"start_time":"2024-11-20T15:18:41.518673","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:22.448201Z","iopub.execute_input":"2024-12-12T14:55:22.448626Z","iopub.status.idle":"2024-12-12T14:55:22.458899Z","shell.execute_reply.started":"2024-12-12T14:55:22.448588Z","shell.execute_reply":"2024-12-12T14:55:22.458182Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"使用ILS-RBFNN进行预测","metadata":{"papermill":{"duration":0.040181,"end_time":"2024-11-20T15:18:41.649244","exception":false,"start_time":"2024-11-20T15:18:41.609063","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\n\nfrom sklearn.cluster import KMeans\n\n\n\n# 激活函数\n\ndef gaussian_rbf(x, c, s):\n\n return np.exp(-np.linalg.norm(x - c) ** 2 / (2 * s ** 2))\n\n\n\n# RBFNN 类\n\nclass RBFNN:\n\n def __init__(self, k=10):\n\n self.k = k # 隐含层节点数\n\n self.centers = None\n\n self.stds = None\n\n self.weights = None\n\n self.classes_ = None\n\n self.label_encoder = None\n\n\n\n def _kmeans(self, X):\n\n kmeans = KMeans(n_clusters=self.k, random_state=0).fit(X)\n\n return kmeans.cluster_centers_\n\n\n\n def _calculate_interpolation_matrix(self, X):\n\n G = np.zeros((X.shape[0], self.k))\n\n for data_point_arg, data_point in enumerate(X):\n\n for center_arg, center in enumerate(self.centers):\n\n G[data_point_arg, center_arg] = gaussian_rbf(data_point, center, self.stds[center_arg])\n\n return G\n\n\n\n def fit(self, X, y):\n\n # 对类别标签进行编码\n\n self.label_encoder = LabelEncoder()\n\n y_encoded = self.label_encoder.fit_transform(y) # 将标签编码成整数\n\n self.classes_ = self.label_encoder.classes_\n\n\n\n self.centers = self._kmeans(X)\n\n d_max = np.max([np.linalg.norm(c1 - c2) for c1 in self.centers for c2 in self.centers])\n\n self.stds = np.repeat(d_max / np.sqrt(2 * self.k), self.k)\n\n G = self._calculate_interpolation_matrix(X)\n\n\n\n # 将标签转换为 one-hot 编码\n\n num_classes = len(self.classes_)\n\n y_one_hot = np.zeros((len(y_encoded), num_classes))\n\n for idx in range(num_classes):\n\n y_one_hot[:, idx] = (y_encoded == idx).astype(float)\n\n\n\n # 计算权重矩阵\n\n self.weights = np.dot(np.linalg.pinv(G), y_one_hot)\n\n\n\n def predict(self, X):\n\n G = self._calculate_interpolation_matrix(X)\n\n predictions = np.dot(G, self.weights)\n\n y_pred_indices = np.argmax(predictions, axis=1)\n\n # 解码类别标签\n\n y_pred = self.label_encoder.inverse_transform(y_pred_indices)\n\n return y_pred\n\n\n\n def predict_proba(self, X):\n\n G = self._calculate_interpolation_matrix(X)\n\n predictions = np.dot(G, self.weights)\n\n # 计算 softmax 概率\n\n exp_preds = np.exp(predictions - np.max(predictions, axis=1, keepdims=True))\n\n probs = exp_preds / np.sum(exp_preds, axis=1, keepdims=True)\n\n return probs\n\n \n\n # 数据清理函数\n\ndef clean_data(X):\n\n # 将 inf 替换为 NaN\n\n X = np.where(np.isinf(X), np.nan, X)\n\n \n\n # 将 X 转换为浮点型以确保可以计算均值\n\n X = X.astype(np.float64)\n\n \n\n # 计算每列的均值（忽略 NaN）\n\n col_means = np.nanmean(X, axis=0)\n\n \n\n # 找到 NaN 的位置\n\n inds = np.where(np.isnan(X))\n\n \n\n # 将 NaN 替换为对应列的均值\n\n X[inds] = np.take(col_means, inds[1])\n\n \n\n return X\n","metadata":{"papermill":{"duration":0.139759,"end_time":"2024-11-20T15:18:41.829109","exception":false,"start_time":"2024-11-20T15:18:41.689350","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:22.459964Z","iopub.execute_input":"2024-12-12T14:55:22.460308Z","iopub.status.idle":"2024-12-12T14:55:22.474726Z","shell.execute_reply.started":"2024-12-12T14:55:22.460272Z","shell.execute_reply":"2024-12-12T14:55:22.473760Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import numpy as np\n\n# import pandas as pd\n\n# from sklearn.model_selection import StratifiedKFold\n\n# from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n\n# from tqdm import tqdm\n\n# import matplotlib.pyplot as plt\n\n# import seaborn as sns\n\n# from scipy.optimize import minimize\n\n# from colorama import Fore, Style\n\n\n\n# # 定义 calculate_qwk 函数（请确保在您的代码中实现）\n\n# def calculate_qwk(y_true, y_pred):\n\n# from sklearn.metrics import cohen_kappa_score\n\n# return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n\n\n# n_splits = 5\n\n# SEED = 42\n\n# num_classes = train_data['sii'].nunique()\n\n# class_numbers = np.arange(num_classes)\n\n\n\n# # 初始化数组用于存储验证集和测试集的预测\n\n# oof_non_rounded = np.zeros(len(train_data), dtype=float)\n\n# oof_rounded = np.zeros(len(train_data), dtype=int)\n\n# test_preds = np.zeros((len(valid_data), n_splits))\n\n\n\n# train_S = []\n\n# test_S = []\n\n\n\n# # 删除 'id' 列，防止泄露（如果存在）\n\n# if 'id' in train_data.columns:\n\n# train_data = train_data.drop(columns=['id'])\n\n\n\n# # 定义并初始化 RBFNN 模型\n\n# model = RBFNN(k=3) # 您可以根据需要调整隐含层节点数 k\n\n\n\n# # 交叉验证\n\n# SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n\n\n# for fold, (train_idx, val_idx) in enumerate(tqdm(SKF.split(train_data, train_data['sii']), desc=\"Training Folds\", total=n_splits)):\n\n# print(f\"\\nFold {fold+1}\")\n\n \n\n# # 划分训练集和验证集，并删除 'sii' 列\n\n# X_train_cv = train_data.iloc[train_idx].drop(columns=['sii']).copy()\n\n# X_val_cv = train_data.iloc[val_idx].drop(columns=['sii']).copy()\n\n# y_train_cv = train_data.iloc[train_idx]['sii'].values\n\n# y_val_cv = train_data.iloc[val_idx]['sii'].values\n\n \n\n# # 将数据转换为 numpy 数组\n\n# X_train_cv = X_train_cv.values\n\n# X_val_cv = X_val_cv.values\n\n \n\n# # 清理数据中的 inf 和 NaN 值\n\n# X_train_cv = clean_data(X_train_cv)\n\n# X_val_cv = clean_data(X_val_cv)\n\n \n\n# # 确保数据中不包含 inf 或 NaN 值\n\n# assert not np.isinf(X_train_cv).any(), \"X_train_cv still contains inf values\"\n\n# assert not np.isnan(X_train_cv).any(), \"X_train_cv still contains NaN values\"\n\n# assert not np.isinf(X_val_cv).any(), \"X_val_cv still contains inf values\"\n\n# assert not np.isnan(X_val_cv).any(), \"X_val_cv still contains NaN values\"\n\n \n\n# # 训练模型\n\n# model.fit(X_train_cv, y_train_cv)\n\n \n\n# # 验证集上的概率预测\n\n# y_val_pred_proba = model.predict_proba(X_val_cv)\n\n# y_val_pred_continuous = np.dot(y_val_pred_proba, class_numbers)\n\n \n\n# # 存储预测结果\n\n# oof_non_rounded[val_idx] = y_val_pred_continuous\n\n# y_val_pred_rounded = np.round(y_val_pred_continuous).astype(int)\n\n# oof_rounded[val_idx] = y_val_pred_rounded\n\n \n\n# # 计算指标\n\n# accuracy = accuracy_score(y_val_cv, y_val_pred_rounded)\n\n# qwk = calculate_qwk(y_val_cv, y_val_pred_rounded)\n\n# print(f\"Fold {fold+1} - Accuracy: {accuracy:.4f}, QWK: {qwk:.4f}\")\n\n \n\n# train_S.append(accuracy)\n\n# test_S.append(qwk)\n\n \n\n# # 在测试集上进行预测\n\n# X_test_cv = valid_data.drop(columns=['id'], errors='ignore').copy()\n\n# X_test_cv = X_test_cv.values\n\n \n\n# # 清理测试集数据\n\n# X_test_cv = clean_data(X_test_cv)\n\n \n\n# y_test_pred_proba = model.predict_proba(X_test_cv)\n\n# y_test_pred_continuous = np.dot(y_test_pred_proba, class_numbers)\n\n# test_preds[:, fold] = y_test_pred_continuous\n\n\n\n# # 输出平均指标\n\n# print(f\"\\nMean Train Accuracy --> {np.mean(train_S):.4f}\")\n\n# print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n\n\n# # 优化 QWK 阈值\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n\n# # 确保 thresholds 单调递增\n\n# thresholds = np.sort(thresholds)\n\n# oof_tuned = np.digitize(oof_non_rounded, bins=thresholds)\n\n# return -calculate_qwk(y_true, oof_tuned)\n\n\n\n# KappaOptimizer = minimize(\n\n# evaluate_predictions,\n\n# x0=[0.5, 1.5, 2.5], # 根据类别数量调整初始阈值\n\n# args=(train_data['sii'].values, oof_non_rounded),\n\n# method='Nelder-Mead'\n\n# )\n\n\n\n# assert KappaOptimizer.success, \"Optimization did not converge.\"\n\n\n\n# # 使用优化后的阈值，确保单调递增\n\n# optimized_thresholds = np.sort(KappaOptimizer.x)\n\n\n\n# def threshold_rounder(values, thresholds):\n\n# return np.digitize(values, bins=thresholds)\n\n\n\n# # 使用优化的阈值对训练集预测进行调整\n\n# oof_tuned = threshold_rounder(oof_non_rounded, optimized_thresholds)\n\n# tKappa = calculate_qwk(train_data['sii'].values, oof_tuned)\n\n\n\n# print(f\"----> || Optimized QWK SCORE :: {Fore.CYAN}{Style.BRIGHT} {tKappa:.3f}{Style.RESET_ALL}\")\n\n\n\n# # 最终测试集的预测值\n\n# final_test_preds = test_preds.mean(axis=1)\n\n# final_test_preds_rounded = threshold_rounder(final_test_preds, optimized_thresholds)\n\n\n\n# # 计算 QWK 分数\n\n# qwk_score = calculate_qwk(valid_sii, final_test_preds_rounded)\n\n# print(f\"QWK 分数: {qwk_score:.4f}\")\n\n\n\n# # 输出分类报告\n\n# print(\"\\nClassification Report:\")\n\n# print(classification_report(valid_sii, final_test_preds_rounded))\n\n\n\n# # 输出混淆矩阵\n\n# conf_matrix = confusion_matrix(valid_sii, final_test_preds_rounded)\n\n# print(\"\\nConfusion Matrix:\")\n\n# print(conf_matrix)\n\n\n\n# # 绘制混淆矩阵热力图\n\n# plt.figure(figsize=(10, 7))\n\n# sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n\n# xticklabels=np.unique(valid_sii),\n\n# yticklabels=np.unique(valid_sii))\n\n# plt.xlabel('Predicted Label')\n\n# plt.ylabel('True Label')\n\n# plt.title('Confusion Matrix')\n\n# plt.show()\n","metadata":{"papermill":{"duration":0.051698,"end_time":"2024-11-20T15:18:41.921269","exception":false,"start_time":"2024-11-20T15:18:41.869571","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:22.481154Z","iopub.execute_input":"2024-12-12T14:55:22.481444Z","iopub.status.idle":"2024-12-12T14:55:22.490272Z","shell.execute_reply.started":"2024-12-12T14:55:22.481417Z","shell.execute_reply":"2024-12-12T14:55:22.489216Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"使用GNN4TDL模型预测","metadata":{"papermill":{"duration":0.04073,"end_time":"2024-11-20T15:18:42.002618","exception":false,"start_time":"2024-11-20T15:18:41.961888","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# import numpy as np\n\n# import pandas as pd\n\n# from sklearn.model_selection import StratifiedKFold\n\n# from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n\n# from sklearn.neighbors import NearestNeighbors\n\n# from sklearn.preprocessing import LabelEncoder\n\n# from tqdm import tqdm\n\n# from scipy.optimize import minimize\n\n# import torch\n\n# import torch.nn.functional as F\n\n# import matplotlib.pyplot as plt\n\n# import seaborn as sns\n\n\n\n# # 定义 calculate_qwk 函数（使用 Quadratic Weighted Kappa）\n\n# from sklearn.metrics import cohen_kappa_score\n\n\n\n# def calculate_qwk(y_true, y_pred):\n\n# return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\n\n\n# # 定义 GNN4TDL 和 GNNModelWrapper（您已提供）\n\n# import torch\n\n# import torch.nn.functional as F\n\n# from torch_geometric.nn import GCNConv\n\n# from sklearn.preprocessing import LabelEncoder\n\n\n\n# class GNN4TDL(torch.nn.Module):\n\n# def __init__(self, input_dim, hidden_dim, output_dim):\n\n# super(GNN4TDL, self).__init__()\n\n# self.conv1 = GCNConv(input_dim, hidden_dim)\n\n# self.conv2 = GCNConv(hidden_dim, output_dim)\n\n\n\n# def forward(self, x, edge_index):\n\n# x = self.conv1(x, edge_index)\n\n# x = F.relu(x)\n\n# x = self.conv2(x, edge_index)\n\n# return F.log_softmax(x, dim=1)\n\n\n\n# # 包装类，适应现有代码\n\n# class GNNModelWrapper:\n\n# def __init__(self, input_dim, hidden_dim=16):\n\n# self.hidden_dim = hidden_dim\n\n# self.model = None\n\n# self.label_encoder = None\n\n\n\n# def fit(self, X, y, edge_index):\n\n# self.label_encoder = LabelEncoder()\n\n# y_encoded = self.label_encoder.fit_transform(y)\n\n# num_classes = len(self.label_encoder.classes_)\n\n \n\n# # 初始化 GNN 模型\n\n# self.model = GNN4TDL(input_dim=X.shape[1], hidden_dim=self.hidden_dim, output_dim=num_classes)\n\n \n\n# # 准备训练数据\n\n# X_tensor = torch.tensor(X, dtype=torch.float)\n\n# y_tensor = torch.tensor(y_encoded, dtype=torch.long)\n\n# # edge_index 已经是 tensor，不需要转换\n\n\n\n# optimizer = torch.optim.Adam(self.model.parameters(), lr=0.01)\n\n# self.model.train()\n\n \n\n# # 训练模型\n\n# for epoch in range(1000):\n\n# optimizer.zero_grad()\n\n# out = self.model(X_tensor, edge_index)\n\n# loss = F.nll_loss(out, y_tensor)\n\n# loss.backward()\n\n# optimizer.step()\n\n# if epoch % 10 == 0:\n\n# print(f\"Epoch {epoch}, Loss: {loss.item():.4f}\")\n\n\n\n# def predict(self, X, edge_index):\n\n# self.model.eval()\n\n# X_tensor = torch.tensor(X, dtype=torch.float)\n\n# # edge_index 已经是 tensor，不需要转换\n\n# with torch.no_grad():\n\n# out = self.model(X_tensor, edge_index)\n\n# pred = out.argmax(dim=1).numpy()\n\n# return self.label_encoder.inverse_transform(pred)\n\n\n\n# def predict_proba(self, X, edge_index):\n\n# self.model.eval()\n\n# X_tensor = torch.tensor(X, dtype=torch.float)\n\n# # edge_index 已经是 tensor，不需要转换\n\n# with torch.no_grad():\n\n# out = self.model(X_tensor, edge_index)\n\n# probs = F.softmax(out, dim=1).numpy()\n\n# return probs\n\n\n\n# # 设置参数\n\n# n_splits = 5\n\n# SEED = 42\n\n# num_classes = train_data['sii'].nunique()\n\n# class_numbers = np.arange(num_classes)\n\n\n\n# # 初始化数组用于存储验证集和测试集的预测\n\n# oof_non_rounded = np.zeros(len(train_data), dtype=float)\n\n# oof_rounded = np.zeros(len(train_data), dtype=int)\n\n# test_preds = np.zeros((len(valid_data), n_splits))\n\n\n\n# train_S = []\n\n# test_S = []\n\n\n\n# # 交叉验证\n\n# SKF = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n\n\n\n# for fold, (train_idx, val_idx) in enumerate(tqdm(SKF.split(train_data, train_data['sii']), desc=\"Training Folds\", total=n_splits)):\n\n# print(f\"\\nFold {fold+1}\")\n\n \n\n# # 划分训练集和验证集\n\n# X_train_cv = train_data.iloc[train_idx].drop(columns=['sii']).copy()\n\n# X_val_cv = train_data.iloc[val_idx].drop(columns=['sii']).copy()\n\n# y_train_cv = train_data.iloc[train_idx]['sii'].values\n\n# y_val_cv = train_data.iloc[val_idx]['sii'].values\n\n \n\n# # 删除 'id' 列，防止泄露\n\n# if 'id' in X_train_cv.columns:\n\n# X_train_cv = X_train_cv.drop(columns=['id'])\n\n# if 'id' in X_val_cv.columns:\n\n# X_val_cv = X_val_cv.drop(columns=['id'])\n\n \n\n# # 将数据转换为 numpy 数组\n\n# X_train_cv = X_train_cv.values\n\n# X_val_cv = X_val_cv.values\n\n \n\n\n\n# # 清理 X_train_cv 和 X_val_cv 中的无穷大和 NaN 值\n\n# X_train_cv = np.nan_to_num(X_train_cv, nan=0.0, posinf=1e10, neginf=-1e10)\n\n# X_val_cv = np.nan_to_num(X_val_cv, nan=0.0, posinf=1e10, neginf=-1e10)\n\n# # 构建训练集的 edge_index（k=5）\n\n# k = 5\n\n# knn = NearestNeighbors(n_neighbors=k+1) # 包含自身\n\n# knn.fit(X_train_cv)\n\n# neighbor_indices = knn.kneighbors(X_train_cv, return_distance=False)\n\n# edge_index_train = [[], []]\n\n# for idx, neighbors in enumerate(neighbor_indices):\n\n# for neighbor in neighbors:\n\n# if idx != neighbor:\n\n# edge_index_train[0].append(idx)\n\n# edge_index_train[1].append(neighbor)\n\n# edge_index_train = torch.tensor(edge_index_train, dtype=torch.long)\n\n \n\n# # 初始化并训练 GNN 模型\n\n# input_dim = X_train_cv.shape[1]\n\n# model = GNNModelWrapper(input_dim=input_dim, hidden_dim=16)\n\n# model.fit(X_train_cv, y_train_cv, edge_index_train)\n\n \n\n# # 构建验证集的 edge_index（k=5）\n\n# knn_val = NearestNeighbors(n_neighbors=k+1)\n\n# knn_val.fit(X_val_cv)\n\n# neighbor_indices_val = knn_val.kneighbors(X_val_cv, return_distance=False)\n\n# edge_index_val = [[], []]\n\n# for idx, neighbors in enumerate(neighbor_indices_val):\n\n# for neighbor in neighbors:\n\n# if idx != neighbor:\n\n# edge_index_val[0].append(idx)\n\n# edge_index_val[1].append(neighbor)\n\n# edge_index_val = torch.tensor(edge_index_val, dtype=torch.long)\n\n \n\n# # 验证集上的概率预测\n\n# y_val_pred_proba = model.predict_proba(X_val_cv, edge_index_val)\n\n# y_val_pred_continuous = np.dot(y_val_pred_proba, class_numbers)\n\n \n\n# # 存储预测结果\n\n# oof_non_rounded[val_idx] = y_val_pred_continuous\n\n# y_val_pred_rounded = np.round(y_val_pred_continuous).astype(int)\n\n# oof_rounded[val_idx] = y_val_pred_rounded\n\n \n\n# # 计算指标\n\n# accuracy = accuracy_score(y_val_cv, y_val_pred_rounded)\n\n# qwk = calculate_qwk(y_val_cv, y_val_pred_rounded)\n\n# print(f\"Fold {fold+1} - Accuracy: {accuracy:.4f}, QWK: {qwk:.4f}\")\n\n \n\n# train_S.append(accuracy)\n\n# test_S.append(qwk)\n\n \n\n# # 在测试集上进行预测\n\n# X_test_cv = valid_data.drop(columns=['id'], errors='ignore').copy()\n\n# X_test_cv = X_test_cv.values\n\n \n\n# X_test_cv = np.nan_to_num(X_test_cv, nan=0.0, posinf=1e10, neginf=-1e10)\n\n# # 构建测试集的 edge_index（k=5）\n\n# knn_test = NearestNeighbors(n_neighbors=k+1)\n\n# knn_test.fit(X_test_cv)\n\n# neighbor_indices_test = knn_test.kneighbors(X_test_cv, return_distance=False)\n\n# edge_index_test = [[], []]\n\n# for idx, neighbors in enumerate(neighbor_indices_test):\n\n# for neighbor in neighbors:\n\n# if idx != neighbor:\n\n# edge_index_test[0].append(idx)\n\n# edge_index_test[1].append(neighbor)\n\n# edge_index_test = torch.tensor(edge_index_test, dtype=torch.long)\n\n \n\n# y_test_pred_proba = model.predict_proba(X_test_cv, edge_index_test)\n\n# y_test_pred_continuous = np.dot(y_test_pred_proba, class_numbers)\n\n# test_preds[:, fold] = y_test_pred_continuous\n\n\n\n# # 输出平均指标\n\n# print(f\"\\nMean Train Accuracy --> {np.mean(train_S):.4f}\")\n\n# print(f\"Mean Validation QWK ---> {np.mean(test_S):.4f}\")\n\n\n\n# # 优化 QWK 阈值\n\n# def evaluate_predictions(thresholds, y_true, oof_non_rounded):\n\n# # 确保 thresholds 单调递增\n\n# thresholds = np.sort(thresholds)\n\n# oof_tuned = np.digitize(oof_non_rounded, bins=thresholds)\n\n# return -calculate_qwk(y_true, oof_tuned)\n\n\n\n# initial_thresholds = np.linspace(0.5, num_classes - 1.5, num=num_classes - 1)\n\n\n\n# KappaOptimizer = minimize(\n\n# evaluate_predictions,\n\n# x0=initial_thresholds,\n\n# args=(train_data['sii'], oof_non_rounded),\n\n# method='Nelder-Mead'\n\n# )\n\n\n\n# assert KappaOptimizer.success, \"Optimization did not converge.\"\n\n\n\n# # 使用优化后的阈值，确保单调递增\n\n# optimized_thresholds = np.sort(KappaOptimizer.x)\n\n\n\n# def threshold_rounder(values, thresholds):\n\n# return np.digitize(values, bins=thresholds)\n\n\n\n# # 使用优化的阈值对训练集预测进行调整\n\n# oof_tuned = threshold_rounder(oof_non_rounded, optimized_thresholds)\n\n# tKappa = calculate_qwk(train_data['sii'], oof_tuned)\n\n\n\n# print(f\"----> || Optimized QWK SCORE :: {tKappa:.3f}\")\n\n\n\n# # 最终测试集的预测值\n\n# final_test_preds = test_preds.mean(axis=1)\n\n# final_test_preds_rounded = threshold_rounder(final_test_preds, optimized_thresholds)\n\n\n\n# # 计算 QWK 分数\n\n# qwk_score = calculate_qwk(valid_sii, final_test_preds_rounded)\n\n# print(f\"QWK 分数: {qwk_score:.4f}\")\n\n\n\n# # 输出分类报告\n\n# print(\"\\nClassification Report:\")\n\n# print(classification_report(valid_sii, final_test_preds_rounded))\n\n\n\n# # 输出混淆矩阵\n\n# conf_matrix = confusion_matrix(valid_sii, final_test_preds_rounded)\n\n# print(\"\\nConfusion Matrix:\")\n\n# print(conf_matrix)\n\n\n\n# # 绘制混淆矩阵热力图\n\n# plt.figure(figsize=(10, 7))\n\n# sns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n\n# xticklabels=np.unique(valid_sii),\n\n# yticklabels=np.unique(valid_sii))\n\n# plt.xlabel('Predicted Label')\n\n# plt.ylabel('True Label')\n\n# plt.title('Confusion Matrix')\n\n# plt.show()\n","metadata":{"papermill":{"duration":0.054333,"end_time":"2024-11-20T15:18:42.097601","exception":false,"start_time":"2024-11-20T15:18:42.043268","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:22.491649Z","iopub.execute_input":"2024-12-12T14:55:22.491956Z","iopub.status.idle":"2024-12-12T14:55:22.509205Z","shell.execute_reply.started":"2024-12-12T14:55:22.491911Z","shell.execute_reply":"2024-12-12T14:55:22.508276Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.039545,"end_time":"2024-11-20T15:18:42.177209","exception":false,"start_time":"2024-11-20T15:18:42.137664","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 导入必要的库\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import KMeans\nfrom scipy.optimize import minimize\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n\n# 导入 TensorFlow 和 Keras 库\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.optimizers import Adam\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.neighbors import NeighborhoodComponentsAnalysis\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nwarnings.filterwarnings('ignore')\n\n# 设置随机种子和参数\nn_splits = 5\nSEED = 42\nnp.random.seed(SEED)\n\n# 假设 train_data, valid_data, test 已经加载\n# train_data, valid_data 中包含 'sii' 列\n\n# ------------------------ 数据预处理部分 ------------------------\n\n# 1. 提取特征并标准化\nfrom sklearn.preprocessing import StandardScaler\n\n# 提取特征列，不包含 'cluster_label' 列\nX_train = train_data.drop(columns=['sii', 'id', 'cluster_label'], errors='ignore')\ny_train = train_data['sii'].values\n\n# 对训练集进行标准化\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\n\n# 对验证集和测试集进行标准化，确保特征列一致\nX_valid = valid_data.drop(columns=['sii', 'id', 'cluster_label'], errors='ignore')\nX_valid_scaled = scaler.transform(X_valid)\n\nX_test = test.drop(columns=['id', 'cluster_label'], errors='ignore')\nX_test_scaled = scaler.transform(X_test)\n\n\n# # 2. 使用 PCA 降维到 10 维\n# pca = PCA(n_components=10, random_state=SEED)\n# X_train_pca = pca.fit_transform(X_train_scaled)\n# X_valid_pca = pca.transform(X_valid_scaled)\n# X_test_pca = pca.transform(X_test_scaled)\n\n# 使用 NCA 降维到 10 维\nnca = NeighborhoodComponentsAnalysis(n_components=6, random_state=SEED)\nX_train_pca = nca.fit_transform(X_train_scaled, y_train) # 使用训练数据和标签进行训练\nX_valid_pca = nca.transform(X_valid_scaled)\nX_test_pca = nca.transform(X_test_scaled)\n\n# # 2. 使用 t-SNE 降维到 10 维\n# # 注意：t-SNE 常用的是 2D 或 3D 降维，10维降维并不常见，建议使用 2D 或 3D 维度\n# tsne = TSNE(n_components=6, random_state=SEED, perplexity=5, method='exact')\n# X_train_pca = tsne.fit_transform(X_train_scaled) # 使用训练数据进行降维\n# X_valid_pca = tsne.fit_transform(X_valid_scaled) # 使用验证集进行降维\n# X_test_pca = tsne.fit_transform(X_test_scaled) # 使用测试集进行降维\n\n# # 定义 Autoencoder 模型\n# input_dim = X_train_scaled.shape[1] # 输入特征的维度\n# encoding_dim = 3 # 将数据压缩到 2 维\n\n# # 编码器\n# input_layer = Input(shape=(input_dim,))\n# encoder = Dense(64, activation='relu')(input_layer)\n# encoder = Dense(32, activation='relu')(encoder)\n# encoder_output = Dense(encoding_dim, activation='linear')(encoder)\n\n# # 解码器\n# decoder = Dense(32, activation='relu')(encoder_output)\n# decoder = Dense(64, activation='relu')(decoder)\n# decoder_output = Dense(input_dim, activation='linear')(decoder)\n\n# # 构建 Autoencoder 模型\n# autoencoder = Model(inputs=input_layer, outputs=decoder_output)\n\n# # 构建 Encoder 模型（用于降维）\n# encoder_model = Model(inputs=input_layer, outputs=encoder_output)\n\n# # 编译模型\n# autoencoder.compile(optimizer='adam', loss='mse')\n\n# # 训练 Autoencoder 模型\n# autoencoder.fit(X_train_scaled, X_train_scaled, epochs=50, batch_size=32, shuffle=True, validation_split=0.2, verbose=1)\n\n# # 使用 Encoder 进行降维\n# X_train_pca = encoder_model.predict(X_train_scaled)\n# X_valid_pca = encoder_model.predict(X_valid_scaled)\n# X_test_pca = encoder_model.predict(X_test_scaled)\n\n# # 使用 LDA 降维到 10 维\n# priors = [0.58,0.26,0.13,0.01]\n# lda = LDA(n_components=3, solver='eigen')\n# X_train_pca = lda.fit_transform(X_train_scaled, y_train) # 使用训练数据和标签进行训练\n# X_valid_pca = lda.transform(X_valid_scaled)\n# X_test_pca = lda.transform(X_test_scaled)\n\n\n\n\n# 3. 使用聚类算法（KMeans）将训练数据分为 4 类\nfrom sklearn.cluster import KMeans\n\nkmeans = KMeans(n_clusters=4, random_state=SEED)\ncluster_labels_train = kmeans.fit_predict(X_train_pca)\n\n# 4. 按照聚类的样本数量从多到少排序，重新赋予聚类标\n# 签为 0、1、2、3\nfrom collections import Counter\n\ncluster_counts = Counter(cluster_labels_train)\nsorted_clusters = [cluster for cluster, _ in cluster_counts.most_common()]\ncluster_mapping = {old_label: new_label for new_label, old_label in enumerate(sorted_clusters)}\ncluster_labels_train_renamed = np.vectorize(cluster_mapping.get)(cluster_labels_train)\n\n# 更新聚类标签\ntrain_data['cluster_label'] = cluster_labels_train_renamed\n\n# 计算训练集上的 QWK 分数\ndef calculate_qwk(y_true, y_pred):\n return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\nqwk_train = calculate_qwk(y_train, cluster_labels_train_renamed)\nprint(f\"Training QWK between 'sii' and cluster labels: {qwk_train:.4f}\")\n\n# 对验证集进行聚类标签预测\ncluster_labels_valid = kmeans.predict(X_valid_pca)\ncluster_labels_valid_renamed = np.vectorize(cluster_mapping.get)(cluster_labels_valid)\n\n# 计算验证集上的 QWK 分数\nqwk_valid = calculate_qwk(valid_sii, cluster_labels_valid_renamed)\nprint(f\"Validation QWK between 'sii' and cluster labels: {qwk_valid:.4f}\")\n\n# 对测试集进行聚类标签预测\ncluster_labels_test = kmeans.predict(X_test_pca)\ncluster_labels_test_renamed = np.vectorize(cluster_mapping.get)(cluster_labels_test)\n\n# 输出验证集的分类报告\nprint(\"\\nValidation Classification Report (Cluster Labels):\")\nprint(classification_report(valid_sii, cluster_labels_valid_renamed))\n\n# 输出验证集的混淆矩阵\nconf_matrix = confusion_matrix(valid_sii, cluster_labels_valid_renamed)\nprint(\"\\nValidation Confusion Matrix (Cluster Labels):\")\nprint(conf_matrix)\n\n# 绘制验证集的混淆矩阵热力图\nplt.figure(figsize=(10, 7))\nsns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n xticklabels=np.unique(cluster_labels_valid_renamed),\n yticklabels=np.unique(cluster_labels_valid_renamed))\nplt.xlabel('Predicted Cluster Label')\nplt.ylabel('True Cluster Label')\nplt.title('Validation Confusion Matrix (Cluster Labels)')\nplt.show()\n\n# 最终测试集的预测值\nfinal_test_preds = test_preds.mean(axis=1)\nfinal_test_preds_rounded = threshold_rounder(final_test_preds, optimized_thresholds)\n\n# 将预测结果与 'id' 对应，生成 submission DataFrame\nsubmission2 = pd.DataFrame({\n 'id': test['id'],\n 'sii': final_test_preds_rounded # 这里的 'sii' 实际上是预测的聚类标签\n})\n\n# 显示提交文件的前几行\nprint(\"\\nSubmission2 Preview:\")\nprint(submission2.head())\n\n# 如果需要，将提交文件保存为 CSV\nsubmission2.to_csv('submission2.csv', index=False)\n\nsubmission2\n","metadata":{"papermill":{"duration":17.549193,"end_time":"2024-11-20T15:18:59.766492","exception":false,"start_time":"2024-11-20T15:18:42.217299","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:22.510694Z","iopub.execute_input":"2024-12-12T14:55:22.511265Z","iopub.status.idle":"2024-12-12T14:55:38.586885Z","shell.execute_reply.started":"2024-12-12T14:55:22.511221Z","shell.execute_reply":"2024-12-12T14:55:38.585879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 导入必要的库\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import KMeans\nfrom scipy.optimize import minimize\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n\n# 导入 TensorFlow 和 Keras 库\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.optimizers import Adam\n\nwarnings.filterwarnings('ignore')\n\n# 设置随机种子和参数\nn_splits = 5\nSEED = 42\nnp.random.seed(SEED)\n\n# 假设 train_data, valid_data, test 已经加载\n# train_data, valid_data 中包含 'sii' 列\n\n# ------------------------ 数据预处理部分 ------------------------\n\n# 1. 提取特征并标准化\nfrom sklearn.preprocessing import StandardScaler\n\n# 提取特征列，不包含 'cluster_label' 列\nX_train = train_data.drop(columns=['sii', 'id', 'cluster_label'], errors='ignore')\ny_train = train_data['sii'].values\n\n# 对训练集进行标准化\nscaler = StandardScaler()\nX_train_scaled = scaler.fit_transform(X_train)\n\n# 对验证集和测试集进行标准化，确保特征列一致\nX_valid = valid_data.drop(columns=['sii', 'id', 'cluster_label'], errors='ignore')\nX_valid_scaled = scaler.transform(X_valid)\n\nX_test = test.drop(columns=['id', 'cluster_label'], errors='ignore')\nX_test_scaled = scaler.transform(X_test)\n\n\n# # 2. 使用 PCA 降维到 10 维\n# pca = PCA(n_components=10, random_state=SEED)\n# X_train_pca = pca.fit_transform(X_train_scaled)\n# X_valid_pca = pca.transform(X_valid_scaled)\n# X_test_pca = pca.transform(X_test_scaled)\n\n# # 使用 NCA 降维到 10 维\n# nca = NeighborhoodComponentsAnalysis(n_components=6, random_state=SEED)\n# X_train_pca = nca.fit_transform(X_train_scaled, y_train) # 使用训练数据和标签进行训练\n# X_valid_pca = nca.transform(X_valid_scaled)\n# X_test_pca = nca.transform(X_test_scaled)\n\n# # 2. 使用 t-SNE 降维到 10 维\n# # 注意：t-SNE 常用的是 2D 或 3D 降维，10维降维并不常见，建议使用 2D 或 3D 维度\n# tsne = TSNE(n_components=6, random_state=SEED, perplexity=5, method='exact')\n# X_train_pca = tsne.fit_transform(X_train_scaled) # 使用训练数据进行降维\n# X_valid_pca = tsne.fit_transform(X_valid_scaled) # 使用验证集进行降维\n# X_test_pca = tsne.fit_transform(X_test_scaled) # 使用测试集进行降维\n\n# # 定义 Autoencoder 模型\n# input_dim = X_train_scaled.shape[1] # 输入特征的维度\n# encoding_dim = 3 # 将数据压缩到 2 维\n\n# # 编码器\n# input_layer = Input(shape=(input_dim,))\n# encoder = Dense(64, activation='relu')(input_layer)\n# encoder = Dense(32, activation='relu')(encoder)\n# encoder_output = Dense(encoding_dim, activation='linear')(encoder)\n\n# # 解码器\n# decoder = Dense(32, activation='relu')(encoder_output)\n# decoder = Dense(64, activation='relu')(decoder)\n# decoder_output = Dense(input_dim, activation='linear')(decoder)\n\n# # 构建 Autoencoder 模型\n# autoencoder = Model(inputs=input_layer, outputs=decoder_output)\n\n# # 构建 Encoder 模型（用于降维）\n# encoder_model = Model(inputs=input_layer, outputs=encoder_output)\n\n# # 编译模型\n# autoencoder.compile(optimizer='adam', loss='mse')\n\n# # 训练 Autoencoder 模型\n# autoencoder.fit(X_train_scaled, X_train_scaled, epochs=50, batch_size=32, shuffle=True, validation_split=0.2, verbose=1)\n\n# # 使用 Encoder 进行降维\n# X_train_pca = encoder_model.predict(X_train_scaled)\n# X_valid_pca = encoder_model.predict(X_valid_scaled)\n# X_test_pca = encoder_model.predict(X_test_scaled)\n\n# 使用 LDA 降维到 10 维\npriors = [0.58,0.26,0.13,0.01]\nlda = LDA(n_components=3, solver='eigen')\nX_train_pca = lda.fit_transform(X_train_scaled, y_train) # 使用训练数据和标签进行训练\nX_valid_pca = lda.transform(X_valid_scaled)\nX_test_pca = lda.transform(X_test_scaled)\n\n\n\n\n# 3. 使用聚类算法（KMeans）将训练数据分为 4 类\nfrom sklearn.cluster import KMeans\n\nkmeans = KMeans(n_clusters=4, random_state=SEED)\ncluster_labels_train = kmeans.fit_predict(X_train_pca)\n\n# 4. 按照聚类的样本数量从多到少排序，重新赋予聚类标\n# 签为 0、1、2、3\nfrom collections import Counter\n\ncluster_counts = Counter(cluster_labels_train)\nsorted_clusters = [cluster for cluster, _ in cluster_counts.most_common()]\ncluster_mapping = {old_label: new_label for new_label, old_label in enumerate(sorted_clusters)}\ncluster_labels_train_renamed = np.vectorize(cluster_mapping.get)(cluster_labels_train)\n\n# 更新聚类标签\ntrain_data['cluster_label'] = cluster_labels_train_renamed\n\n# 计算训练集上的 QWK 分数\ndef calculate_qwk(y_true, y_pred):\n return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n\nqwk_train = calculate_qwk(y_train, cluster_labels_train_renamed)\nprint(f\"Training QWK between 'sii' and cluster labels: {qwk_train:.4f}\")\n\n# 对验证集进行聚类标签预测\ncluster_labels_valid = kmeans.predict(X_valid_pca)\ncluster_labels_valid_renamed = np.vectorize(cluster_mapping.get)(cluster_labels_valid)\n\n# 计算验证集上的 QWK 分数\nqwk_valid = calculate_qwk(valid_sii, cluster_labels_valid_renamed)\nprint(f\"Validation QWK between 'sii' and cluster labels: {qwk_valid:.4f}\")\n\n# 对测试集进行聚类标签预测\ncluster_labels_test = kmeans.predict(X_test_pca)\ncluster_labels_test_renamed = np.vectorize(cluster_mapping.get)(cluster_labels_test)\n\n# 输出验证集的分类报告\nprint(\"\\nValidation Classification Report (Cluster Labels):\")\nprint(classification_report(valid_sii, cluster_labels_valid_renamed))\n\n# 输出验证集的混淆矩阵\nconf_matrix = confusion_matrix(valid_sii, cluster_labels_valid_renamed)\nprint(\"\\nValidation Confusion Matrix (Cluster Labels):\")\nprint(conf_matrix)\n\n# 绘制验证集的混淆矩阵热力图\nplt.figure(figsize=(10, 7))\nsns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues',\n xticklabels=np.unique(cluster_labels_valid_renamed),\n yticklabels=np.unique(cluster_labels_valid_renamed))\nplt.xlabel('Predicted Cluster Label')\nplt.ylabel('True Cluster Label')\nplt.title('Validation Confusion Matrix (Cluster Labels)')\nplt.show()\n\n# 最终测试集的预测值\nfinal_test_preds = test_preds.mean(axis=1)\nfinal_test_preds_rounded = threshold_rounder(final_test_preds, optimized_thresholds)\n\n# 将预测结果与 'id' 对应，生成 submission DataFrame\nsubmission3 = pd.DataFrame({\n 'id': test['id'],\n 'sii': final_test_preds_rounded # 这里的 'sii' 实际上是预测的聚类标签\n})\n\n# 显示提交文件的前几行\nprint(\"\\nSubmission3 Preview:\")\nprint(submission3.head())\n\nsubmission3\n\n# 如果需要，将提交文件保存为 CSV\n# submission.to_csv('submission.csv', index=False)\n","metadata":{"papermill":{"duration":0.600571,"end_time":"2024-11-20T15:19:00.411525","exception":false,"start_time":"2024-11-20T15:18:59.810954","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:38.588619Z","iopub.execute_input":"2024-12-12T14:55:38.589020Z","iopub.status.idle":"2024-12-12T14:55:39.227227Z","shell.execute_reply.started":"2024-12-12T14:55:38.588977Z","shell.execute_reply":"2024-12-12T14:55:39.226237Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1 = submission1\nsub2 = submission2\nsub3 = submission3\n\nsub1 = sub1.sort_values(by='id').reset_index(drop=True)\nsub2 = sub2.sort_values(by='id').reset_index(drop=True)\nsub3 = sub3.sort_values(by='id').reset_index(drop=True)\n\ncombined = pd.DataFrame({\n 'id': sub1['id'],\n 'sii_1': sub1['sii'],\n 'sii_2': sub2['sii'],\n 'sii_3': sub3['sii']\n})\n\ndef majority_vote(row):\n return row.mode()[0]\n\ncombined['final_sii'] = combined[['sii_1', 'sii_2', 'sii_3']].apply(majority_vote, axis=1)\n\nfinal_submission = combined[['id', 'final_sii']].rename(columns={'final_sii': 'sii'})\n\nfinal_submission.to_csv('submission1.csv', index=False)\n\nprint(\"Majority voting completed and saved to 'Final_Submission.csv'\")\n\nfinal_submission","metadata":{"papermill":{"duration":0.074803,"end_time":"2024-11-20T15:19:00.530160","exception":false,"start_time":"2024-11-20T15:19:00.455357","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T14:55:39.228477Z","iopub.execute_input":"2024-12-12T14:55:39.228765Z","iopub.status.idle":"2024-12-12T14:55:39.253201Z","shell.execute_reply.started":"2024-12-12T14:55:39.228737Z","shell.execute_reply":"2024-12-12T14:55:39.252187Z"}},"outputs":[],"execution_count":null}]}