{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":50160,"databundleVersionId":7921029}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nfrom pathlib import Path\nimport subprocess\nimport os\nimport gc\nfrom glob import glob\nimport glob  \n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom datetime import datetime\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nROOT = '../input/home-credit-credit-risk-model-stability'\n\nfrom sklearn.model_selection import TimeSeriesSplit, GroupKFold, StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, RegressorMixin\nfrom sklearn.metrics import roc_auc_score\nimport lightgbm as lgb\n\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.impute import KNNImputer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-04T10:24:13.397852Z","iopub.execute_input":"2024-05-04T10:24:13.398199Z","iopub.status.idle":"2024-05-04T10:24:20.567962Z","shell.execute_reply.started":"2024-05-04T10:24:13.39817Z","shell.execute_reply":"2024-05-04T10:24:20.566997Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Pipeline:\n\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n        return df\n\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))  #!!?\n                df = df.with_columns(pl.col(col).dt.total_days()) # t - t-1\n        df = df.drop(\"date_decision\", \"MONTH\")\n        return df\n\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n                if isnull > 0.7:\n                    df = df.drop(col)\n        \n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n        \n        return df\n\n\nclass Aggregator:\n    @staticmethod\n    def num_expr(df):\n        # 筛选出列名以\"P\"或\"A\"结尾的数值列\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n\n        # 为每个数值列构建一个计算最大值的聚合表达式\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        #得到的结果是一个列表，这个列表中的每个元素都是一个Polars表达式。\n        #这些表达式定义了对原始数据框架中的某些列进行聚合操作（在这个例子中是计算最大值）并为结果赋予新的列名。\n        #列表中的每个表达式对应于原数据中的一个列，\n        #执行聚合操作后，每个表达式会生成一个新的列，列名为原列名前加上\"max_\"前缀。\n\n        return expr_max\n\n    @staticmethod\n    def date_expr(df):\n        # 筛选出列名以\"D\"结尾的日期列\n        cols = [col for col in df.columns if col[-1] in (\"D\",)]\n\n        # 为每个日期列构建一个计算最大值的聚合表达式\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def str_expr(df):\n        # 筛选出列名以\"M\"结尾的字符串列\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n\n        # 为每个字符串列构建一个计算最大值的聚合表达式\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def other_expr(df):\n        # 筛选出列名以\"T\"或\"L\"结尾的其他类型列\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n\n        # 为这些其他类型列构建一个计算最大值的聚合表达式\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def count_expr(df):\n        # 筛选出列名中包含\"num_group\"的列，这些列可能涉及分组计数或索引\n        cols = [col for col in df.columns if \"num_group\" in col]\n        # 为这些分组计数或索引列构建一个计算最大值的聚合表达式\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n\n    @staticmethod\n    def get_exprs(df):\n        # 组合上述所有类型的聚合表达式\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n        # 返回包含所有聚合表达式的列表\n        return exprs\n\ndef read_file(path, depth=None):\n    df = pl.read_parquet(path)  # または pl.read_csv(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    if depth in [1, 2]:\n        df = df.groupby(\"case_id\").agg(Aggregator.get_exprs(df))\n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    # 正しい glob 関数の使用: glob.glob() を使用してファイルパターンに一致する全ファイルを取得\n    for path in glob.glob(str(regex_path)):\n        df = pl.read_parquet(path)  # または pl.read_csv(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        if depth in [1, 2]:\n            df = df.groupby(\"case_id\").agg(Aggregator.get_exprs(df))\n        chunks.append(df)\n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    return df\n\ndef feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n    df_base = df_base.pipe(Pipeline.handle_dates)\n    return df_base\n\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    return df_data, cat_cols\n\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:24:20.570006Z","iopub.execute_input":"2024-05-04T10:24:20.570897Z","iopub.status.idle":"2024-05-04T10:24:20.607829Z","shell.execute_reply.started":"2024-05-04T10:24:20.570859Z","shell.execute_reply":"2024-05-04T10:24:20.606856Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRAIN_DIR       = ROOT + \"/parquet_files/\" + \"train/\"\nTEST_DIR        = ROOT + \"/parquet_files/\" + \"test/\"\nTRAIN_DIR","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:24:20.60909Z","iopub.execute_input":"2024-05-04T10:24:20.609448Z","iopub.status.idle":"2024-05-04T10:24:20.62471Z","shell.execute_reply.started":"2024-05-04T10:24:20.609416Z","shell.execute_reply":"2024-05-04T10:24:20.623796Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndata_store = {\n    \"df_base\": read_file(Path(TRAIN_DIR) / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(Path(TRAIN_DIR) / \"train_static_cb_0.parquet\"),\n        read_files(Path(TRAIN_DIR) / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(Path(TRAIN_DIR) / \"train_applprev_1_*.parquet\", 1),\n        read_file(Path(TRAIN_DIR) / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(Path(TRAIN_DIR) / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(Path(TRAIN_DIR) / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(Path(TRAIN_DIR) / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(Path(TRAIN_DIR) / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(Path(TRAIN_DIR) / \"train_other_1.parquet\", 1),\n        read_file(Path(TRAIN_DIR) / \"train_person_1.parquet\", 1),\n        read_file(Path(TRAIN_DIR) / \"train_deposit_1.parquet\", 1),\n        read_file(Path(TRAIN_DIR) / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(Path(TRAIN_DIR) / \"train_credit_bureau_b_2.parquet\", 2),\n        read_files(Path(TRAIN_DIR) / \"train_credit_bureau_a_2_*.parquet\", 2),\n        read_file(Path(TRAIN_DIR) / \"train_applprev_2.parquet\", 2),\n        read_file(Path(TRAIN_DIR) / \"train_person_2.parquet\", 2)\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:24:20.626785Z","iopub.execute_input":"2024-05-04T10:24:20.627086Z","iopub.status.idle":"2024-05-04T10:26:25.043123Z","shell.execute_reply.started":"2024-05-04T10:24:20.627056Z","shell.execute_reply":"2024-05-04T10:26:25.042142Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndf_train = feature_eng(**data_store)\nprint(\"train data shape:\\t\", df_train.shape)\ndel data_store\ngc.collect()\ndf_train = df_train.pipe(Pipeline.filter_cols)\ndf_train, cat_cols = to_pandas(df_train)\ndf_train = reduce_mem_usage(df_train)\nprint(\"train data shape:\\t\", df_train.shape)\nnums=df_train.select_dtypes(exclude='category').columns\n\nfrom itertools import combinations, permutations\n#df_train=df_train[nums]\nnans_df = df_train[nums].isna()\nnans_groups={}\nfor col in nums:\n    cur_group = nans_df[col].sum()\n    try:\n        nans_groups[cur_group].append(col)\n    except:\n        nans_groups[cur_group]=[col]\ndel nans_df; x=gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:26:25.04606Z","iopub.execute_input":"2024-05-04T10:26:25.046321Z","iopub.status.idle":"2024-05-04T10:27:04.277894Z","shell.execute_reply.started":"2024-05-04T10:26:25.046299Z","shell.execute_reply":"2024-05-04T10:27:04.276823Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 各カラムの欠損値の数を計算\nmissing_counts = df_train.isna().sum()\n\n# 欠損値が少ない順にカラム名をソート\nsorted_columns = missing_counts.sort_values().index.tolist()\n\n# 100個ずつカラム名を出力\nfor i in range(0, len(sorted_columns), 100):\n    print(sorted_columns[i:i+100],\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:27:04.27908Z","iopub.execute_input":"2024-05-04T10:27:04.279349Z","iopub.status.idle":"2024-05-04T10:27:05.409521Z","shell.execute_reply.started":"2024-05-04T10:27:04.279325Z","shell.execute_reply":"2024-05-04T10:27:05.408579Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\ndef reduce_group(grps):\n    use = []\n    for g in grps:\n        mx = 0; vx = g[0]\n        for gg in g:\n            n = df_train[gg].nunique()\n            if n>mx:\n                mx = n\n                vx = gg\n            #print(str(gg)+'-'+str(n),', ',end='')\n        use.append(vx)\n        #print()\n    print('Use these',use)\n    return use\n\ndef group_columns_by_correlation(matrix, threshold=0.8):\n    # 计算列之间的相关性\n    correlation_matrix = matrix.corr()\n\n    # 分组列\n    groups = []\n    remaining_cols = list(matrix.columns)\n    while remaining_cols:\n        col = remaining_cols.pop(0)\n        group = [col]\n        correlated_cols = [col]\n        for c in remaining_cols:\n            if correlation_matrix.loc[col, c] >= threshold:\n                group.append(c)\n                correlated_cols.append(c)\n        groups.append(group)\n        remaining_cols = [c for c in remaining_cols if c not in correlated_cols]\n    \n    return groups\n\nuses=[]\nfor k,v in nans_groups.items():\n    if len(v)>1:\n            Vs = nans_groups[k]\n            #cross_features=list(combinations(Vs, 2))\n            #make_corr(Vs)\n            grps= group_columns_by_correlation(df_train[Vs], threshold=0.8)\n            use=reduce_group(grps)\n            uses=uses+use\n            #make_corr(use)\n    else:\n        uses=uses+v\n    print('####### NAN count =',k)\nprint(uses)\nprint(len(uses))\nuses=uses+list(df_train.select_dtypes(include='category').columns)\nprint(len(uses))\ndf_train=df_train[uses]","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:27:05.410708Z","iopub.execute_input":"2024-05-04T10:27:05.411022Z","iopub.status.idle":"2024-05-04T10:27:05.439059Z","shell.execute_reply.started":"2024-05-04T10:27:05.410997Z","shell.execute_reply":"2024-05-04T10:27:05.438111Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample = pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/sample_submission.csv\")\ndevice='gpu'\n#n_samples=200000\nn_est=6000\nDRY_RUN = True if sample.shape[0] == 10 else False   \nif DRY_RUN:\n    device='cpu'\n    df_train = df_train.iloc[:60000]\n    #n_samples=10000\n    n_est=600\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:27:05.44037Z","iopub.execute_input":"2024-05-04T10:27:05.44065Z","iopub.status.idle":"2024-05-04T10:27:05.454629Z","shell.execute_reply.started":"2024-05-04T10:27:05.440627Z","shell.execute_reply":"2024-05-04T10:27:05.453686Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TEST_DIR + \"test_base.parquet\"),\n    \"depth_0\": [\n        read_file(TEST_DIR + \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR + \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR + \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR + \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR + \"test_tax_registry_b_1.parquet\", 1),\n        read_file(TEST_DIR + \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR + \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR + \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR + \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR + \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR + \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR + \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR + \"test_credit_bureau_b_2.parquet\", 2),\n        read_files(TEST_DIR + \"test_credit_bureau_a_2_*.parquet\", 2),\n        read_file(TEST_DIR + \"test_applprev_2.parquet\", 2),\n        read_file(TEST_DIR + \"test_person_2.parquet\", 2)\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:27:05.456002Z","iopub.execute_input":"2024-05-04T10:27:05.456699Z","iopub.status.idle":"2024-05-04T10:27:05.768618Z","shell.execute_reply.started":"2024-05-04T10:27:05.456663Z","shell.execute_reply":"2024-05-04T10:27:05.767626Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\nprint(\"test data shape:\\t\", df_test.shape)\ndel data_store\ngc.collect()\ndf_test = df_test.select([col for col in df_train.columns if col != \"target\"])\nprint(\"train data shape:\\t\", df_train.shape)\nprint(\"test data shape:\\t\", df_test.shape)\n\ndf_test, cat_cols = to_pandas(df_test, cat_cols)\ndf_test = reduce_mem_usage(df_test)\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:27:05.77202Z","iopub.execute_input":"2024-05-04T10:27:05.772307Z","iopub.status.idle":"2024-05-04T10:27:06.282766Z","shell.execute_reply.started":"2024-05-04T10:27:05.772277Z","shell.execute_reply":"2024-05-04T10:27:06.281864Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = df_train[\"target\"]\nweeks = df_train[\"WEEK_NUM\"]\ndf_train= df_train.drop(columns=[\"target\", \"case_id\", \"WEEK_NUM\"])\ncv = StratifiedGroupKFold(n_splits=5, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:27:06.28377Z","iopub.execute_input":"2024-05-04T10:27:06.284062Z","iopub.status.idle":"2024-05-04T10:27:06.341139Z","shell.execute_reply.started":"2024-05-04T10:27:06.284036Z","shell.execute_reply":"2024-05-04T10:27:06.340318Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train[cat_cols] = df_train[cat_cols].astype(str)\ndf_test[cat_cols] = df_test[cat_cols].astype(str)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:27:06.342261Z","iopub.execute_input":"2024-05-04T10:27:06.342547Z","iopub.status.idle":"2024-05-04T10:27:06.589285Z","shell.execute_reply.started":"2024-05-04T10:27:06.342523Z","shell.execute_reply":"2024-05-04T10:27:06.587991Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {\n    \"boosting_type\": \"gbdt\",\n    \"objective\": \"binary\",\n    \"metric\": \"auc\",\n    \"max_depth\": 10,  \n    \"learning_rate\": 0.05,\n    \"n_estimators\": 2000,\n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"verbose\": -1,\n    \"random_state\": 42,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 10,\n    \"extra_trees\":True,\n    'num_leaves':64,\n    \"device\": device, \n    \"verbose\": -1,\n}\n\nparams2 = {\n    \"booster\": \"gbtree\",\n    \"objective\": \"binary:logistic\",\n    \"eval_metric\": \"auc\",\n    \"max_depth\": 10,\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 1000,\n    \"colsample_bytree\": 0.8,\n    \"colsample_bynode\": 0.8,\n    \"alpha\": 0.1,  \n    \"lambda\": 10,  \n    \"tree_method\": 'gpu_hist' if device == 'gpu' else 'auto',\n    \"random_state\": 42,\n    \"verbosity\": 0,\n    \"enable_categorical\":True,\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:27:06.591006Z","iopub.execute_input":"2024-05-04T10:27:06.591366Z","iopub.status.idle":"2024-05-04T10:27:06.601091Z","shell.execute_reply.started":"2024-05-04T10:27:06.591333Z","shell.execute_reply":"2024-05-04T10:27:06.599724Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%time\nfrom catboost import CatBoostClassifier, Pool\nimport xgboost as xgb\n\n# 特徴量重要度のDataFrameを保存するリスト\nfeature_importances_cat = []\nfeature_importances_lgb = []\nfeature_importances_xgb = []\n\nfor idx_train, idx_valid in cv.split(df_train, y, groups=weeks):\n    X_train, y_train = df_train.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = df_train.iloc[idx_valid], y.iloc[idx_valid]\n\n    # CatBoost\n    train_pool = Pool(X_train, y_train, cat_features=cat_cols)\n    val_pool = Pool(X_valid, y_valid, cat_features=cat_cols)\n    clf = CatBoostClassifier(\n        eval_metric='AUC',\n        task_type='GPU',\n        learning_rate=0.03,\n        iterations=n_est,\n        random_seed=3107  # Corrected the placement of random_seed\n    )\n    clf.fit(train_pool, eval_set=val_pool, verbose=300)\n    importances = clf.get_feature_importance(prettified=True)\n    feature_importances_cat.append(pd.DataFrame(importances).head(30))\n\n    # LightGBM\n    X_train[cat_cols] = X_train[cat_cols].astype(\"category\")\n    X_valid[cat_cols] = X_valid[cat_cols].astype(\"category\")\n    model = lgb.LGBMClassifier(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        callbacks=[lgb.log_evaluation(200), lgb.early_stopping(100)]\n    )\n#     fitted_models_lgb.append(model)\n    importances = pd.DataFrame({\n        'Feature': X_train.columns,\n        'Importance': model.feature_importances_\n    }).sort_values(by='Importance', ascending=False)\n    feature_importances_lgb.append(importances.head(30))\n\n    # XGBoost\n    model2 = xgb.XGBClassifier(**params2)\n    model2.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        early_stopping_rounds=100, verbose=False\n    )\n#     fitted_models_xgb.append(model2)\n    importances = pd.DataFrame({\n        'Feature': X_train.columns,\n        'Importance': model2.feature_importances_\n    }).sort_values(by='Importance', ascending=False)\n    feature_importances_xgb.append(importances.head(30))\n\n    del clf, model, model2\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:28:35.488197Z","iopub.execute_input":"2024-05-04T10:28:35.488743Z","iopub.status.idle":"2024-05-04T10:38:00.14883Z","shell.execute_reply.started":"2024-05-04T10:28:35.488713Z","shell.execute_reply":"2024-05-04T10:38:00.147844Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CatBoostのフィーチャーIDのユニークなリストを取得\nunique_features_cat = pd.concat(feature_importances_cat[0:5]).drop_duplicates('Feature Id')\nprint(\"Unique Feature IDs for CatBoost from folds 6 to 10:\")\nprint(unique_features_cat['Feature Id'])\n\n# LightGBMのフィーチャーIDのユニークなリストを取得\nunique_features_lgb = pd.concat(feature_importances_lgb[0:5]).drop_duplicates('Feature')\nprint(\"\\nUnique Feature IDs for LightGBM from folds 6 to 10:\")\nprint(unique_features_lgb['Feature'])\n\n# XGBoostのフィーチャーIDのユニークなリストを取得\nunique_features_xgb = pd.concat(feature_importances_xgb[0:5]).drop_duplicates('Feature')\nprint(\"\\nUnique Feature IDs for XGBoost from folds 6 to 10:\")\nprint(unique_features_xgb['Feature'])\n\n# CatBoost、LightGBM、XGBoostから抽出されたフィーチャー名のリストを連結\nall_unique_features = pd.concat([\n    unique_features_cat['Feature Id'],\n    unique_features_lgb['Feature'],\n    unique_features_xgb['Feature']\n])\n\n# ユニークなフィーチャー名を抽出\nall_unique_features = all_unique_features.drop_duplicates().reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:41:22.671847Z","iopub.execute_input":"2024-05-04T10:41:22.67272Z","iopub.status.idle":"2024-05-04T10:41:22.688544Z","shell.execute_reply.started":"2024-05-04T10:41:22.672686Z","shell.execute_reply":"2024-05-04T10:41:22.687609Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# all_unique_features のカラム名を確認\n# print(\"all_unique_features column names:\", all_unique_features.tolist())\n\n# 実際のカテゴリカルカラムをDataFrameから抽出\nactual_categorical_cols = df_train.select_dtypes(include=['object', 'category']).columns.tolist()\n# print(\"Actual categorical columns in DataFrame:\", actual_categorical_cols)\n\n# all_unique_features と実際のカテゴリカルカラムの共通項を探す\ncommon_cat_features = [col for col in actual_categorical_cols if col in all_unique_features.tolist()]\n# print(\"Common categorical columns in 'all_unique_features':\", common_cat_features)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:41:22.689967Z","iopub.execute_input":"2024-05-04T10:41:22.690253Z","iopub.status.idle":"2024-05-04T10:41:22.815081Z","shell.execute_reply.started":"2024-05-04T10:41:22.690223Z","shell.execute_reply":"2024-05-04T10:41:22.814081Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%time\nfrom catboost import CatBoostClassifier, Pool\nimport xgboost as xgb\n\nfitted_models_cat = []\nfitted_models_lgb = []\nfitted_models_xgb = []\n\ncv_scores_cat = []\ncv_scores_lgb = []\ncv_scores_xgb = []\n\ndf_train_filtered = df_train[all_unique_features].copy()\n\nfor idx_train, idx_valid in cv.split(df_train_filtered, y, groups=weeks):\n    X_train, y_train = df_train_filtered.iloc[idx_train], y.iloc[idx_train]\n    X_valid, y_valid = df_train_filtered.iloc[idx_valid], y.iloc[idx_valid]\n\n    # CatBoost\n    train_pool = Pool(X_train, y_train, cat_features=common_cat_features)\n    val_pool = Pool(X_valid, y_valid, cat_features=common_cat_features)\n    clf = CatBoostClassifier(\n        eval_metric='AUC',\n        task_type='GPU',\n        learning_rate=0.03,\n        iterations=n_est,\n        random_seed=3107  # Corrected the placement of random_seed\n    )\n    clf.fit(train_pool, eval_set=val_pool, verbose=300)\n    fitted_models_cat.append(clf)\n    y_pred_valid = clf.predict_proba(X_valid)[:, 1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_cat.append(auc_score)\n\n    # LightGBM\n    X_train[common_cat_features] = X_train[common_cat_features].astype(\"category\")\n    X_valid[common_cat_features] = X_valid[common_cat_features].astype(\"category\")\n    model = lgb.LGBMClassifier(**params)\n    model.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        callbacks=[lgb.log_evaluation(200), lgb.early_stopping(100)]\n    )\n    fitted_models_lgb.append(model)\n    y_pred_valid = model.predict_proba(X_valid)[:, 1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_lgb.append(auc_score)\n\n    # XGBoost\n    model2 = xgb.XGBClassifier(**params2)\n    model2.fit(\n        X_train, y_train,\n        eval_set=[(X_valid, y_valid)],\n        early_stopping_rounds=100, verbose=False\n    )\n    y_pred_valid = model2.predict_proba(X_valid)[:, 1]\n    auc_score = roc_auc_score(y_valid, y_pred_valid)\n    cv_scores_xgb.append(auc_score)\n\n    del clf, model, model2\n    gc.collect()\n\nprint(\"CV AUC scores for CatBoost: \", cv_scores_cat)\nprint(\"Maximum CV AUC score for CatBoost: \", max(cv_scores_cat))\n\nprint(\"CV AUC scores for LightGBM: \", cv_scores_lgb)\nprint(\"Maximum CV AUC score for LightGBM: \", max(cv_scores_lgb))\n\nprint(\"CV AUC scores for XGBoost: \", cv_scores_xgb)\nprint(\"Maximum CV AUC score for XGBoost: \", max(cv_scores_xgb))","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:42:27.528485Z","iopub.execute_input":"2024-05-04T10:42:27.528843Z","iopub.status.idle":"2024-05-04T10:47:48.396331Z","shell.execute_reply.started":"2024-05-04T10:42:27.528815Z","shell.execute_reply":"2024-05-04T10:47:48.395396Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class VotingModel(BaseEstimator, RegressorMixin):\n    def __init__(self, estimators):\n        super().__init__()\n        self.estimators = estimators\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def predict(self, X):\n        y_preds = [estimator.predict(X) for estimator in self.estimators]\n        return np.mean(y_preds, axis=0)\n    \n    def predict_proba(self, X):\n        \n        y_preds = [estimator.predict_proba(X) for estimator in self.estimators[:5]]\n        \n        X[common_cat_features] = X[common_cat_features].astype(\"category\")\n        y_preds += [estimator.predict_proba(X) for estimator in self.estimators[5:10]]\n        y_preds+=y_preds #tang trong so\n        y_preds += [estimator.predict_proba(X) for estimator in self.estimators[10:]]\n        print(len(y_preds))\n        return np.mean(y_preds, axis=0)\n\nmodel = VotingModel(fitted_models_cat+fitted_models_lgb+fitted_models_xgb)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:47:48.398264Z","iopub.execute_input":"2024-05-04T10:47:48.398892Z","iopub.status.idle":"2024-05-04T10:47:48.407717Z","shell.execute_reply.started":"2024-05-04T10:47:48.398856Z","shell.execute_reply":"2024-05-04T10:47:48.406639Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import cross_val_predict\nfrom catboost import CatBoostClassifier, Pool\nfrom lightgbm import LGBMClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import roc_auc_score\nimport pandas as pd\n\n# 第一层模型\nmodels = [\n    ('CatBoost', CatBoostClassifier(eval_metric='AUC', task_type='GPU', learning_rate=0.03, iterations=n_est, random_seed=3107)),\n    ('LightGBM', LGBMClassifier(**params)),\n    ('XGBoost', XGBClassifier(**params2))\n]\n\n# 第二层模型\nfrom sklearn.ensemble import GradientBoostingClassifier\n\nparams = {\n    'n_estimators': 12,\n    'learning_rate': 0.1,\n    'max_depth': 3,\n    'min_samples_split': 3,\n    'min_samples_leaf': 1\n}\n\nmeta_model = GradientBoostingClassifier(**params)\n\n\n\n# 存储第一层模型和相应的AUC分数\nfitted_models_cb = []\nfitted_models_lgb = []\nfitted_models_xgb = []\ncv_scores_cb = []\ncv_scores_lgb = []\ncv_scores_xgb = []\n\n# 存储第一层模型的预测结果\nmeta_features = pd.DataFrame(index=df_train_filtered.index, columns=['CatBoost', 'LightGBM', 'XGBoost'])\n\nfor name, model in models:\n    for idx_train, idx_valid in cv.split(df_train_filtered, y, groups=weeks):\n        X_train, y_train = df_train_filtered.iloc[idx_train], y.iloc[idx_train]\n        X_valid, y_valid = df_train_filtered.iloc[idx_valid], y.iloc[idx_valid]\n        \n        if name == 'CatBoost':\n            X_train[common_cat_features] = X_train[common_cat_features].astype(str)\n            X_valid[common_cat_features] = X_valid[common_cat_features].astype(str)\n            train_pool = Pool(X_train, y_train, cat_features=common_cat_features)\n            val_pool = Pool(X_valid, y_valid, cat_features=common_cat_features)\n            model.fit(train_pool, eval_set=val_pool, verbose=False)\n            y_pred_valid = model.predict_proba(val_pool)[:, 1]\n            fitted_models_cb.append(model)\n            auc_score = roc_auc_score(y_valid, y_pred_valid)\n            cv_scores_cb.append(auc_score)\n        elif name == 'LightGBM':\n            X_train[common_cat_features] = X_train[common_cat_features].astype('category')\n            X_valid[common_cat_features] = X_valid[common_cat_features].astype('category')\n            model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], callbacks=[lgb.log_evaluation(200), lgb.early_stopping(100)])\n            fitted_models_lgb.append(model)\n            y_pred_valid = model.predict_proba(X_valid)[:, 1]\n            auc_score = roc_auc_score(y_valid, y_pred_valid)\n            cv_scores_lgb.append(auc_score)\n        else:  # XGBoost\n            X_train[common_cat_features] = X_train[common_cat_features].astype('category')\n            X_valid[common_cat_features] = X_valid[common_cat_features].astype('category')\n            model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], early_stopping_rounds=100, verbose=False)\n            fitted_models_xgb.append(model)\n            y_pred_valid = model.predict_proba(X_valid)[:, 1]\n            auc_score = roc_auc_score(y_valid, y_pred_valid)\n            cv_scores_xgb.append(auc_score)\n\n        meta_features.loc[X_valid.index, name] = y_pred_valid","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:54:05.01241Z","iopub.execute_input":"2024-05-04T10:54:05.013137Z","iopub.status.idle":"2024-05-04T10:58:52.656775Z","shell.execute_reply.started":"2024-05-04T10:54:05.013103Z","shell.execute_reply":"2024-05-04T10:58:52.655893Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meta_model.fit(meta_features, y)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:58:52.658711Z","iopub.execute_input":"2024-05-04T10:58:52.659268Z","iopub.status.idle":"2024-05-04T10:58:53.540364Z","shell.execute_reply.started":"2024-05-04T10:58:52.659236Z","shell.execute_reply":"2024-05-04T10:58:53.539467Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# all_unique_features が pandas Series の場合、リストに変換\nif isinstance(all_unique_features, pd.Series):\n    all_unique_features = all_unique_features.tolist()\n\n# case_id をカラムリストに追加\ncolumns_to_copy = all_unique_features + [\"case_id\"]\n\n# df_test から指定したカラムを選択してコピー\ndf_test_filtered = df_test[columns_to_copy].copy()\n# df_test_filtered = df_test_filtered.drop(columns=[\"WEEK_NUM\"])\ndf_test_filtered = df_test_filtered.set_index(\"case_id\")","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:58:53.54156Z","iopub.execute_input":"2024-05-04T10:58:53.541885Z","iopub.status.idle":"2024-05-04T10:58:53.559795Z","shell.execute_reply.started":"2024-05-04T10:58:53.541859Z","shell.execute_reply":"2024-05-04T10:58:53.558909Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_meta_features = pd.DataFrame(index=df_test_filtered.index, columns=['CatBoost', 'LightGBM', 'XGBoost'])","metadata":{"execution":{"iopub.status.busy":"2024-05-04T10:58:53.561843Z","iopub.execute_input":"2024-05-04T10:58:53.562161Z","iopub.status.idle":"2024-05-04T10:58:53.567575Z","shell.execute_reply.started":"2024-05-04T10:58:53.562136Z","shell.execute_reply":"2024-05-04T10:58:53.566416Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for model in fitted_models_cat:\n    df_test_filtered[common_cat_features] = df_test_filtered[common_cat_features].astype(str)\n    y_pred_test = model.predict_proba(df_test_filtered)[:, 1]\n    test_meta_features['CatBoost'] = test_meta_features['CatBoost'].add(y_pred_test, fill_value=0)\n\ntest_meta_features['CatBoost'] /= len(fitted_models_cat)\n\n# LightGBM\nfor model in fitted_models_lgb:\n    df_test_filtered[common_cat_features] = df_test_filtered[common_cat_features].astype(\"category\")\n    y_pred_test = model.predict_proba(df_test_filtered)[:, 1]\n    test_meta_features['LightGBM'] = test_meta_features['LightGBM'].add(y_pred_test, fill_value=0)\n\ntest_meta_features['LightGBM'] /= len(fitted_models_lgb)\n\n# XGBoost\nfor model in fitted_models_xgb:\n    df_test_filtered[common_cat_features] = df_test_filtered[common_cat_features].astype(\"category\")\n    y_pred_test = model.predict_proba(df_test_filtered)[:, 1]\n    test_meta_features['XGBoost'] = test_meta_features['XGBoost'].add(y_pred_test, fill_value=0)\n\ntest_meta_features['XGBoost'] /= len(fitted_models_xgb)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T11:00:38.585163Z","iopub.execute_input":"2024-05-04T11:00:38.585541Z","iopub.status.idle":"2024-05-04T11:00:39.00372Z","shell.execute_reply.started":"2024-05-04T11:00:38.585511Z","shell.execute_reply":"2024-05-04T11:00:38.998856Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_meta_features","metadata":{"execution":{"iopub.status.busy":"2024-05-04T11:00:39.005919Z","iopub.execute_input":"2024-05-04T11:00:39.00684Z","iopub.status.idle":"2024-05-04T11:00:39.023656Z","shell.execute_reply.started":"2024-05-04T11:00:39.006797Z","shell.execute_reply":"2024-05-04T11:00:39.022556Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = pd.Series(meta_model.predict_proba(test_meta_features)[:, 1], index=df_test_filtered.index)","metadata":{"execution":{"iopub.status.busy":"2024-05-04T11:00:39.025271Z","iopub.execute_input":"2024-05-04T11:00:39.025873Z","iopub.status.idle":"2024-05-04T11:00:39.033096Z","shell.execute_reply.started":"2024-05-04T11:00:39.025841Z","shell.execute_reply":"2024-05-04T11:00:39.032017Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_subm = pd.read_csv(ROOT + \"/sample_submission.csv\")\ndf_subm = df_subm.set_index(\"case_id\")\n\ndf_subm[\"score\"] = y_pred\ndf_subm.to_csv(\"submission.csv\")\ndf_subm","metadata":{"execution":{"iopub.status.busy":"2024-05-04T11:00:39.035604Z","iopub.execute_input":"2024-05-04T11:00:39.036446Z","iopub.status.idle":"2024-05-04T11:00:39.056581Z","shell.execute_reply.started":"2024-05-04T11:00:39.036411Z","shell.execute_reply":"2024-05-04T11:00:39.055713Z"},"trusted":true},"outputs":[],"execution_count":null}]}