{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8389815,"sourceType":"datasetVersion","datasetId":4990248},{"sourceId":8392553,"sourceType":"datasetVersion","datasetId":4992391},{"sourceId":8396838,"sourceType":"datasetVersion","datasetId":4995398},{"sourceId":8402708,"sourceType":"datasetVersion","datasetId":4999793},{"sourceId":8422327,"sourceType":"datasetVersion","datasetId":5014302},{"sourceId":8428534,"sourceType":"datasetVersion","datasetId":5019029},{"sourceId":8430222,"sourceType":"datasetVersion","datasetId":5020319}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"papermill":{"default_parameters":{},"duration":254.282286,"end_time":"2024-05-16T11:24:25.100557","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-05-16T11:20:10.818271","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#!/usr/bin/env python\n# coding: utf-8\n\n# In[1]:\n\n\nimport sys\nfrom pathlib import Path\nimport subprocess\nimport os\nimport gc\nfrom glob import glob\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom datetime import datetime\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pickle\nimport warnings\nwarnings.filterwarnings('ignore')\n\nROOT = '/kaggle/input/home-credit-credit-risk-model-stability'\n\nfrom sklearn.model_selection import TimeSeriesSplit, GroupKFold, StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\n\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.impute import KNNImputer\n\n# In[2]:\n\n\nclass Pipeline:\n\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n        return df\n\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))  #!!?\n                df = df.with_columns(pl.col(col).dt.total_days()) # t - t-1\n        df = df.drop( \"MONTH\")\n        return df\n\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n                if isnull > 0.95:\n                    df = df.drop(col)\n        \n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n        \n        return df\n\n\nclass Aggregator:\n    #Please add or subtract features yourself, be aware that too many features will take up too much space.\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        \n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        return expr_max +expr_last+expr_mean\n    \n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n        return  expr_max +expr_last+expr_mean\n    \n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        #expr_count = [pl.count(col).alias(f\"count_{col}\") for col in cols]\n        return  expr_max +expr_last#+expr_count\n    \n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        return  expr_max +expr_last\n    \n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols] \n        #expr_min = [pl.min(col).alias(f\"min_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        #expr_first = [pl.first(col).alias(f\"first_{col}\") for col in cols]\n        return  expr_max +expr_last\n    \n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs\n\ndef read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    if depth in [1,2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df)) \n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    \n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        chunks.append(df)\n    \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    return df\n","metadata":{"execution":{"iopub.execute_input":"2024-05-16T11:20:13.530912Z","iopub.status.busy":"2024-05-16T11:20:13.530588Z","iopub.status.idle":"2024-05-16T11:20:19.066482Z","shell.execute_reply":"2024-05-16T11:20:19.065568Z"},"papermill":{"duration":5.544666,"end_time":"2024-05-16T11:20:19.068988","exception":false,"start_time":"2024-05-16T11:20:13.524322","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.003967,"end_time":"2024-05-16T11:20:19.077992","exception":false,"start_time":"2024-05-16T11:20:19.074025","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef feature_eng(df_base, depth_0, depth_1, depth_2):\n    '''df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )'''\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n    df_base = df_base.pipe(Pipeline.handle_dates)\n    return df_base\n\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    return df_data, cat_cols\n\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\n# In[3]:\n\n\nROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\n\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR        = ROOT / \"parquet_files\" / \"test\"\n\n# In[4]:\n\ndata_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n        read_file(TRAIN_DIR / \"train_applprev_2.parquet\", 2),\n        read_file(TRAIN_DIR / \"train_person_2.parquet\", 2)\n    ]\n}\n\n# In[5]:\n\ndf_train = feature_eng(**data_store)\nprint(\"train data shape:\\t\", df_train.shape)\ndel data_store\ngc.collect()\ndf_train = df_train.pipe(Pipeline.filter_cols)\n\ndf_train, cat_cols = to_pandas(df_train)\n\n# txt_file_path = '/kaggle/input/hacktest-516/ensemble_cat_cols.txt'\n# with open(txt_file_path, 'w') as f:\n#     for column_name in cat_cols:\n#         f.write(\"%s\\n\" % column_name)\n        \ndate = df_train['date_decision']\ndel df_train['date_decision']\nprint(\"date_decision save\")\n\ndf_train = reduce_mem_usage(df_train)\nprint(\"train data shape:\\t\", df_train.shape)\n\nnums=df_train.select_dtypes(exclude='category').columns\nfrom itertools import combinations, permutations\n#df_train=df_train[nums]\nnans_df = df_train[nums].isna()\nnans_groups={}\nfor col in nums:\n    cur_group = nans_df[col].sum()\n    try:\n        nans_groups[cur_group].append(col)\n    except:\n        nans_groups[cur_group]=[col]\ndel nans_df; x=gc.collect()\n\ndef reduce_group(grps):\n    use = []\n    for g in grps:\n        mx = 0; vx = g[0]\n        for gg in g:\n            n = df_train[gg].nunique()\n            if n>mx:\n                mx = n\n                vx = gg\n            #print(str(gg)+'-'+str(n),', ',end='')\n        use.append(vx)\n        #print()\n    print('Use these',use)\n    return use\n\ndef group_columns_by_correlation(matrix, threshold=0.8):\n    # 计算列之间的相关性\n    correlation_matrix = matrix.corr()\n\n    # 分组列\n    groups = []\n    remaining_cols = list(matrix.columns)\n    while remaining_cols:\n        col = remaining_cols.pop(0)\n        group = [col]\n        correlated_cols = [col]\n        for c in remaining_cols:\n            if correlation_matrix.loc[col, c] >= threshold:\n                group.append(c)\n                correlated_cols.append(c)\n        groups.append(group)\n        remaining_cols = [c for c in remaining_cols if c not in correlated_cols]\n    \n    return groups\n\nuses=[]\nfor k,v in nans_groups.items():\n    if len(v)>1:\n            Vs = nans_groups[k]\n            #cross_features=list(combinations(Vs, 2))\n            #make_corr(Vs)\n            grps= group_columns_by_correlation(df_train[Vs], threshold=0.8)\n            use=reduce_group(grps)\n            uses=uses+use\n            #make_corr(use)\n    else:\n        uses=uses+v\n    print('####### NAN count =',k)\nprint(uses)\nprint(len(uses))\nuses=uses+list(df_train.select_dtypes(include='category').columns)\nprint(len(uses))\ndf_train=df_train[uses]\n\n# In[6]:\n\n\nsample = pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/sample_submission.csv\")\ndevice='gpu'\n#n_samples=200000\n\nn_est=6000\n\n#DRY_RUN = True if sample.shape[0] == 10 else False   \n#if DRY_RUN:\n#    device='cpu'\n#    df_train = df_train.iloc[:10000]\n#    n_samples=10000\n#    n_est=600\n\nprint(device)","metadata":{"execution":{"iopub.execute_input":"2024-05-16T11:20:19.088593Z","iopub.status.busy":"2024-05-16T11:20:19.087937Z","iopub.status.idle":"2024-05-16T11:24:19.403522Z","shell.execute_reply":"2024-05-16T11:24:19.402457Z"},"papermill":{"duration":240.323219,"end_time":"2024-05-16T11:24:19.405614","exception":false,"start_time":"2024-05-16T11:20:19.082395","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# In[7]:\n\n\ndata_store = {\n    \"df_base\": read_file(TEST_DIR / \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n        read_files(TEST_DIR / \"test_credit_bureau_a_2_*.parquet\", 2),\n        read_file(TEST_DIR / \"test_applprev_2.parquet\", 2),\n        read_file(TEST_DIR / \"test_person_2.parquet\", 2)\n    ]\n}\n\n# In[8]:\ndf_test = feature_eng(**data_store)\nprint(\"test data shape:\\t\", df_test.shape)\ndel data_store\ngc.collect()\n\n#numtest = df_test[['WEEK_NUM']]\ndatetest = df_test[['date_decision']]\ndf_test = df_test.drop(columns=['date_decision'])\nprint(\"test_date_decision save\")\n\n\n\ntxt_file_path = '/kaggle/input/hackmodel-516noon/ensemble_column_names.txt'\ncolumn_names = []\nwith open(txt_file_path, 'r') as f:\n    for line in f:\n        column_names.append(line.strip())\n\n   \n\ndf_test = df_test.select([col for col in column_names if col not in [\"target\"]])\nprint(\"train data shape:\\t\", df_train.shape)\nprint(\"test data shape:\\t\", df_test.shape)\n\n\n\ntxt_file_path = '/kaggle/input/hackmodel-516noon/ensemble_cat_cols.txt' \ncat_cols = []\nwith open(txt_file_path, 'r') as f:\n    for line in f:\n        cat_cols.append(line.strip())\n\ndf_test, cat_cols = to_pandas(df_test, cat_cols)\ndf_test = reduce_mem_usage(df_test)\ndf_test['date_decision'] = datetest['date_decision']\n#df_test['WEEK_NUM'] = numtest['WEEK_NUM']\n\n\ngc.collect()\n\n\n# In[10]:\n#读取数据 \ndf_test2 = df_test.copy()\n#数据预处理\ndf_test = df_test.drop(columns=['date_decision'])\ndf_test[cat_cols] = df_test[cat_cols].astype(str)  \n \n\n# In[11]:\n\n\nparams = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 10,  \n    \"learning_rate\": 0.05,\n    \"n_estimators\": 2000,  \n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\":True,\n    'num_leaves':64,\n    \"device\": device, \n    \"verbose\": -1,\n}\n\n# In[12]:\n\n\nfrom catboost import CatBoostClassifier, Pool\nimport pickle\nfitted_models_cat = []\nfitted_models_lgb = []\nn_fold=5\ncv_scores_cat = []\ncv_scores_lgb = []\n\n# 加载每个训练好的模型文件到fitted_models列表中\nfor i in range(n_fold):\n    with open(f'/kaggle/input/hackmodel-516noon/cat_model_{i}.pkl', 'rb') as fin:\n        loaded_model = pickle.load(fin)\n        fitted_models_cat.append(loaded_model)\n\nfor i in range(n_fold):\n    with open(f'/kaggle/input/hackmodel-516noon/model_{i}.pkl', 'rb') as fin:\n        fitted_models_lgb.append(pickle.load(fin))\n\n# In[13]:\n\n\nclass VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n    \n    def predict_proba(self, X):\n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators[:5]]\n        X[cat_cols] = X[cat_cols].astype(\"category\")\n        y_preds += [estimator.predict_proba(X) for estimator in self.estimators[5:]]\n        \n        return np.mean(y_preds, axis=0)\n    \n\n    \n#放入测试集进行预测 \nmodel = VotingModel(fitted_models_cat+fitted_models_lgb)\ndf_test = df_test.drop(columns=[\"WEEK_NUM\"])\ndf_test = df_test.set_index(\"case_id\")\ny_pred = pd.Series(model.predict_proba(df_test)[:, 1], index=df_test.index)\ndf_subm = pd.read_csv(ROOT / \"sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\ndf_subm[\"score\"] = y_pred\ndf_subm\n","metadata":{"execution":{"iopub.execute_input":"2024-05-16T11:24:19.421972Z","iopub.status.busy":"2024-05-16T11:24:19.421620Z","iopub.status.idle":"2024-05-16T11:24:23.877661Z","shell.execute_reply":"2024-05-16T11:24:23.876666Z"},"papermill":{"duration":4.466677,"end_time":"2024-05-16T11:24:23.879773","exception":false,"start_time":"2024-05-16T11:24:19.413096","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_subm.columns)","metadata":{"execution":{"iopub.execute_input":"2024-05-16T11:24:23.896715Z","iopub.status.busy":"2024-05-16T11:24:23.896428Z","iopub.status.idle":"2024-05-16T11:24:23.900847Z","shell.execute_reply":"2024-05-16T11:24:23.900031Z"},"papermill":{"duration":0.015354,"end_time":"2024-05-16T11:24:23.903095","exception":false,"start_time":"2024-05-16T11:24:23.887741","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#预测周数较小的\n\n\n#数据预处理\nif 'date_decision' in df_test2.columns:\n    print(\"'date_decision' 列存在df_test2\")\nelse:\n    print(\"'date_decision' 列不存在df_test2\")\n    \n# 计算日期差值\nbirthdate_first_valid = pd.Timestamp('2019-01-01 00:00:00')\ndf_test2.loc[:, \"days_difference\"] = (df_test2[\"date_decision\"] - birthdate_first_valid).dt.days\n\n# 删除原始日期列\ndf_test2 = df_test2.drop(columns=[\"date_decision\"])\n\n# 读取类别特征列\ntxt_file_path = '/kaggle/input/hackmodel-516noon/ensemble_cat_cols.txt'\ncat_cols = []\nwith open(txt_file_path, 'r') as f:\n    for line in f:\n        cat_cols.append(line.strip())\n\ndf_test2[cat_cols] = df_test2[cat_cols].astype(str)\n\nX_test = df_test2.drop(columns=[\"case_id\", \"WEEK_NUM\"])\n\n# 确保 X_test 中的类别特征列与训练时使用的列一致\nX_test[cat_cols] = X_test[cat_cols].astype(str)\n\n# 读入模型\nimport joblib\nmodel_path = '/kaggle/input/hack-model-516-night/lgb_models_online_516_night.pkl'\nmodel = joblib.load(model_path)\n\nclass VotingModel1(BaseEstimator, RegressorMixin):\n    def __init__(self, model):\n        super().__init__()\n        self.model = model\n\n    def fit(self, X, y=None):\n        # 因为不需要训练过程，所以这里什么也不做\n        pass\n\n    def predict(self, X):\n        return self.model.predict(X)\n\n    def predict_proba(self, X):\n        return self.model.predict_proba(X)\n\n\n \n# 进行预测\nhack_model1 = VotingModel1(loaded_model)\ny_pred_test = pd.DataFrame()\ny_pred_test['result'] = hack_model1.predict_proba(X_test)[:, 1]\nprint(\" predict  finish#####################################################################################################\")\n\n\n\n# # 找到result最小的一半样本的索引\n# num_samples = y_pred_test.shape[0]\n# half_samples = num_samples // 2\n\n\n\n# # 按result排序，并获取最小的一半样本的索引\n# sorted_indices = y_pred_test['result'].nsmallest(half_samples).index\n\n# # 获取相应的case_id\n# case_ids = df_test.iloc[sorted_indices, :].index\n# print(\"Smallest case_ids:\",case_ids)\n\n\n# # 修改 score_bias 的值\n# df_subm['score_bias'] = 0\n# df_subm.loc[df_subm.index.isin(case_ids), 'score_bias'] = 0.02\n# df_subm['score'] = df_subm['score'] - df_subm['score_bias']\n\n# # 将 score 限制在 0 到 1 之间\n# df_subm['score'] = np.clip(df_subm['score'], 0, 1)\n\n# # 保存结果到 CSV 文件\n# df_subm.to_csv(\"submission.csv\")\n\n# # 删除 score_bias 列\n# del df_subm['score_bias']\n# df_subm\n","metadata":{"execution":{"iopub.execute_input":"2024-05-16T11:24:23.919467Z","iopub.status.busy":"2024-05-16T11:24:23.919214Z","iopub.status.idle":"2024-05-16T11:24:24.104338Z","shell.execute_reply":"2024-05-16T11:24:24.103171Z"},"papermill":{"duration":0.195915,"end_time":"2024-05-16T11:24:24.106575","exception":false,"start_time":"2024-05-16T11:24:23.910660","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_test","metadata":{"execution":{"iopub.execute_input":"2024-05-16T11:24:24.125433Z","iopub.status.busy":"2024-05-16T11:24:24.125137Z","iopub.status.idle":"2024-05-16T11:24:24.133710Z","shell.execute_reply":"2024-05-16T11:24:24.132849Z"},"papermill":{"duration":0.020192,"end_time":"2024-05-16T11:24:24.135933","exception":false,"start_time":"2024-05-16T11:24:24.115741","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n# 计算要修改的样本数量\nnum_samples = y_pred_test.shape[0]\nnum_samples_to_modify = num_samples // 2\n\n# 找到result最小的一半样本的索引\nsorted_indices = np.argsort(y_pred_test['result'])\nprint(\"sorted_indices \",sorted_indices)\nindices_to_modify = sorted_indices[:num_samples_to_modify]\nprint(\"indices_to_modify:\",indices_to_modify)\n\n# 修改对应样本的score值\ndf_subm.iloc[indices_to_modify, df_subm.columns.get_loc(\"score\")] -= 0.02\n\n# 确保score值不会小于0\ndf_subm['score'] = df_subm['score'].clip(lower=0)\n\n# 保存修改后的DataFrame\ndf_subm.to_csv(\"submission.csv\")\ndf_subm","metadata":{"execution":{"iopub.execute_input":"2024-05-16T11:24:24.153745Z","iopub.status.busy":"2024-05-16T11:24:24.153467Z","iopub.status.idle":"2024-05-16T11:24:24.170080Z","shell.execute_reply":"2024-05-16T11:24:24.169160Z"},"papermill":{"duration":0.027515,"end_time":"2024-05-16T11:24:24.172145","exception":false,"start_time":"2024-05-16T11:24:24.144630","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}