{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":80344,"databundleVersionId":8608242,"sourceType":"competition"},{"sourceId":8498845,"sourceType":"datasetVersion","datasetId":5071608},{"sourceId":8501122,"sourceType":"datasetVersion","datasetId":5073506},{"sourceId":33095,"sourceType":"modelInstanceVersion","modelInstanceId":27710,"modelId":39234},{"sourceId":33096,"sourceType":"modelInstanceVersion","modelInstanceId":27711,"modelId":39234}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":277.286256,"end_time":"2024-05-18T08:04:38.27647","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-05-18T08:00:00.990214","version":"2.5.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys  # System-specific parameters and functions\nimport subprocess  # Spawn new processes, connect to their input/output/error pipes, and obtain their return codes\nimport os  # Operating system dependent functionality\nimport gc  # Garbage Collector interface\nfrom pathlib import Path  # Object-oriented filesystem paths\nfrom glob import glob  # Unix style pathname pattern expansion\nimport numpy as np  # Fundamental package for scientific computing with Python\nimport pandas as pd  # Powerful data structures for data manipulation and analysis\nimport polars as pl  # Fast DataFrame library implemented in Rust\nfrom datetime import datetime  # Basic date and time types\nimport seaborn as sns  # Statistical data visualization\nimport matplotlib.pyplot as plt  # MATLAB-like plotting framework\nimport joblib  # Save and load Python objects\nimport warnings  # Warning control\nwarnings.filterwarnings('ignore')  # Ignore warnings\nfrom sklearn.base import BaseEstimator, RegressorMixin  # Base classes for all estimators in scikit-learn\nfrom sklearn.metrics import roc_auc_score  # ROC AUC score\nimport lightgbm as lgb  # LightGBM: Gradient boosting framework\nfrom sklearn.model_selection import TimeSeriesSplit, GroupKFold, StratifiedGroupKFold  # Cross-validation strategies\nfrom imblearn.over_sampling import SMOTE  # Oversampling technique for imbalanced datasets\nfrom sklearn.preprocessing import OrdinalEncoder  # Encode categorical features as an integer array\nfrom sklearn.impute import KNNImputer  # Imputation for completing missing values using k-Nearest Neighbors","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:25:05.008042Z","iopub.execute_input":"2024-05-23T22:25:05.008989Z","iopub.status.idle":"2024-05-23T22:25:12.710863Z","shell.execute_reply.started":"2024-05-23T22:25:05.008918Z","shell.execute_reply":"2024-05-23T22:25:12.709851Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ROOT = '/kaggle/input/home-credit-credit-risk-model-stability'  # Setting the root directory path","metadata":{"papermill":{"duration":159.410917,"end_time":"2024-05-18T08:02:43.44582","exception":false,"start_time":"2024-05-18T08:00:04.034903","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-05-23T22:25:12.712608Z","iopub.execute_input":"2024-05-23T22:25:12.713304Z","iopub.status.idle":"2024-05-23T22:25:12.717834Z","shell.execute_reply.started":"2024-05-23T22:25:12.713261Z","shell.execute_reply":"2024-05-23T22:25:12.716846Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Pipeline:\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n        return df\n\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))  #!!?\n                df = df.with_columns(pl.col(col).dt.total_days()) # t - t-1\n        df = df.drop(\"date_decision\", \"MONTH\")\n        return df\n\n    def filter_cols(df):\n        \n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n        \n        return df\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:25:12.719398Z","iopub.execute_input":"2024-05-23T22:25:12.720139Z","iopub.status.idle":"2024-05-23T22:25:12.732242Z","shell.execute_reply.started":"2024-05-23T22:25:12.720114Z","shell.execute_reply":"2024-05-23T22:25:12.731272Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Aggregator:\n    #Please add or subtract features yourself, be aware that too many features will take up too much space.\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return expr_max\n    \n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return  expr_max\n    \n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return  expr_max\n    \n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        return  expr_max \n    \n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols] \n        return  expr_max\n    \n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:25:12.734155Z","iopub.execute_input":"2024-05-23T22:25:12.734425Z","iopub.status.idle":"2024-05-23T22:25:12.750407Z","shell.execute_reply.started":"2024-05-23T22:25:12.734404Z","shell.execute_reply":"2024-05-23T22:25:12.749322Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    if depth in [1,2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df)) \n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    \n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        chunks.append(df)\n    \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    return df\n\ndef feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n    df_base = df_base.pipe(Pipeline.handle_dates)\n    return df_base\n\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    return df_data, cat_cols\n\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2    \n    return df\n","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:25:12.751466Z","iopub.execute_input":"2024-05-23T22:25:12.751960Z","iopub.status.idle":"2024-05-23T22:25:12.777433Z","shell.execute_reply.started":"2024-05-23T22:25:12.751912Z","shell.execute_reply":"2024-05-23T22:25:12.776419Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"\n# TEST_DIR        = ROOT / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:28:43.202321Z","iopub.execute_input":"2024-05-23T22:28:43.202690Z","iopub.status.idle":"2024-05-23T22:28:43.208026Z","shell.execute_reply.started":"2024-05-23T22:28:43.202659Z","shell.execute_reply":"2024-05-23T22:28:43.207000Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:28:44.401374Z","iopub.execute_input":"2024-05-23T22:28:44.401740Z","iopub.status.idle":"2024-05-23T22:29:56.383736Z","shell.execute_reply.started":"2024-05-23T22:28:44.401710Z","shell.execute_reply":"2024-05-23T22:29:56.382876Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from itertools import combinations, permutations","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:30:07.690055Z","iopub.execute_input":"2024-05-23T22:30:07.690423Z","iopub.status.idle":"2024-05-23T22:30:07.694847Z","shell.execute_reply.started":"2024-05-23T22:30:07.690394Z","shell.execute_reply":"2024-05-23T22:30:07.693886Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\ndel data_store\ngc.collect()\ndf_train = df_train.pipe(Pipeline.filter_cols)\ndf_train, cat_cols = to_pandas(df_train)\ndf_train = reduce_mem_usage(df_train)\nnums=df_train.select_dtypes(exclude='category').columns\nnans_df = df_train[nums].isna()\nnans_groups={}\n\nfor col in nums:\n    cur_group = nans_df[col].sum()\n    try:\n        nans_groups[cur_group].append(col)\n    except:\n        nans_groups[cur_group]=[col]\n\ndel nans_df; x=gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:30:08.088331Z","iopub.execute_input":"2024-05-23T22:30:08.088673Z","iopub.status.idle":"2024-05-23T22:30:55.877126Z","shell.execute_reply.started":"2024-05-23T22:30:08.088645Z","shell.execute_reply":"2024-05-23T22:30:55.875891Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"caseid = df_train[\"case_id\"]","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:42:09.913385Z","iopub.execute_input":"2024-05-23T22:42:09.914172Z","iopub.status.idle":"2024-05-23T22:42:09.918511Z","shell.execute_reply.started":"2024-05-23T22:42:09.914140Z","shell.execute_reply":"2024-05-23T22:42:09.917427Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:42:20.257460Z","iopub.execute_input":"2024-05-23T22:42:20.258167Z","iopub.status.idle":"2024-05-23T22:42:20.735567Z","shell.execute_reply.started":"2024-05-23T22:42:20.258133Z","shell.execute_reply":"2024-05-23T22:42:20.734548Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature selection","metadata":{}},{"cell_type":"code","source":"!pip install -q google-generativeai","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:25:19.445155Z","iopub.execute_input":"2024-05-23T22:25:19.446022Z","iopub.status.idle":"2024-05-23T22:25:32.396345Z","shell.execute_reply.started":"2024-05-23T22:25:19.445981Z","shell.execute_reply":"2024-05-23T22:25:32.395157Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport json\nimport google.generativeai as genai\nimport os\nimport time\nfrom tqdm import tqdm\n\ngenai.configure(api_key=\"AIzaSyBN-99XrOUFmBhlXzm9Fci2_w4u7WSzdc8\")\nmodel = genai.GenerativeModel('gemini-1.5-pro-latest')\nstability_generation_config = genai.GenerationConfig(temperature=0, top_p=1, top_k=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:25:32.398449Z","iopub.execute_input":"2024-05-23T22:25:32.398766Z","iopub.status.idle":"2024-05-23T22:25:33.326772Z","shell.execute_reply.started":"2024-05-23T22:25:32.398738Z","shell.execute_reply":"2024-05-23T22:25:33.325839Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"domain = \"financial\"\ntask_description = \"\"\"\nThe absence of a credit history might mean a lot of things, including young age or a preference for cash. Without traditional data, someone with little to no credit history is likely to be denied. Consumer finance providers must accurately determine which clients can repay a loan and which cannot and data is key. If data science could help better predict one’s repayment capabilities, loans might become more accessible to those who may benefit from them the most.\nCurrently, consumer finance providers use various statistical and machine learning methods to predict loan risk. These models are generally called scorecards. In the real world, clients' behaviors change constantly, so every scorecard must be updated regularly, which takes time. The scorecard's stability in the future is critical, as a sudden drop in performance means that loans will be issued to worse clients on average. The core of the issue is that loan providers aren't able to spot potential problems any sooner than the first due dates of those loans are observable. Given the time it takes to redevelop, validate, and implement the scorecard, stability is highly desirable. There is a trade-off between the stability of the model and its performance, and a balance must be reached before deployment.\nFounded in 1997, competition host Home Credit is an international consumer finance provider focusing on responsible lending primarily to people with little or no credit history. Home Credit broadens financial inclusion for the unbanked population by creating a positive and safe borrowing experience. We previously ran a competition with Kaggle that you can see here.\nYour work in helping to assess potential clients' default risks will enable consumer finance providers to accept more loan applications. This may improve the lives of people who have historically been denied due to lack of credit history.\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:25:33.327991Z","iopub.execute_input":"2024-05-23T22:25:33.328815Z","iopub.status.idle":"2024-05-23T22:25:33.335411Z","shell.execute_reply.started":"2024-05-23T22:25:33.328769Z","shell.execute_reply":"2024-05-23T22:25:33.334358Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_df = pd.read_csv(\"/kaggle/input/home-credit-credit-risk-model-stability/feature_definitions.csv\")\navailable_columns = features_df['Variable'].tolist()\n\nvalid_columns = list(set(available_columns).intersection(set(df_train.columns)))\nother_columns = list(set(df_train.columns) - set(valid_columns))","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:25:33.337622Z","iopub.execute_input":"2024-05-23T22:25:33.337951Z","iopub.status.idle":"2024-05-23T22:25:34.084517Z","shell.execute_reply.started":"2024-05-23T22:25:33.337908Z","shell.execute_reply":"2024-05-23T22:25:34.083054Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_answer_from_gemini(*prompts, **generate_config):\n    prompt =  \"\\n\".join(map(str, prompts))\n    response = model.generate_content(prompt, **generate_config)\n    return response.text\n\ndef get_descriptive_columns(columns_list, dataframe):\n    descriptions = []\n    columns = []\n    available_columns = dataframe['Variable'].tolist()\n    for column in columns_list:\n        if column not in available_columns:\n            continue\n        matched_df = dataframe[dataframe['Variable'] == column]\n        \n        columns.append(matched_df['Variable'].iloc[0])\n        descriptions.append(matched_df['Description'].iloc[0])\n    \n    return zip(columns, descriptions)\n\ndef parse_json(result):\n    result = result.replace(\"\\'\", \"\\\"\")\n    if \"`\" not in result:\n        return json.loads(result)\n    \n    result = result.replace(\"```json\", \"```\")\n    match_result = re.search(\"`([^`]+)`\", result)\n    if match_result == None:\n        return {}\n    return json.loads(match_result.group(1))\n\n# def reduce_group(grps):\n#     use = []\n#     for g in grps:\n#         mx = 0; vx = g[0]\n#         for gg in g:\n#             n = df_train[gg].nunique()\n#             if n>mx:\n#                 mx = n\n#                 vx = gg\n#         use.append(vx)\n#     return use\n\n# def group_columns_by_correlation(matrix, threshold=0.8):\n#     correlation_matrix = matrix.corr()\n#     groups = []\n#     remaining_cols = list(matrix.columns)\n#     while remaining_cols:\n#         col = remaining_cols.pop(0)\n#         group = [col]\n#         correlated_cols = [col]\n#         for c in remaining_cols:\n#             if correlation_matrix.loc[col, c] >= threshold:\n#                 group.append(c)\n#                 correlated_cols.append(c)\n#         groups.append(group)\n#         remaining_cols = [c for c in remaining_cols if c not in correlated_cols]\n    \n#     return groups\n\n# def get_missing_value_guide(dataframe, columns_pair, domain, task):\n#     system_prompt = f\"You're data analytics expert especially in {domain} domain\"\n#     instruction_template = \"\"\"\n#     Here is the task\n#     {task}\n\n#     If the given column has an {missing_percentage} of missing values\n#     Column: {column_name}\n#     Description: {description}\n\n#     Based on your domain knowledge and data analytics skill\n#     Question: 1. What are these missing values mean?\n#     Question: 2. Does it make sense if we impute this missing value? (purpose: make better data)\n\n#     Please answer two question shortly and in this format only!\n\n#     The missing value percent is <missing_percent>\n#     Meaning of missing value: <answer1>\n#     Should we impute?: <answer2>\n#     \"\"\".replace(\"{task}\", task)\n    \n#     answer_list = []\n    \n#     for column, description in tqdm(columns_pair):\n#         missing_percentage = int(round(df_train[column].isna().sum() / df_train.shape[0], 2) * 100)\n#         prompt = instruction_template.replace('{column_name}', column).replace(\"{description}\", description).replace(\"{missing_percentage}\", str(missing_percentage))\n#         answer = get_answer_from_gemini(\n#             system_prompt, prompt, generation_config=stability_generation_config)\n#         answer_list.append(answer)\n\n#     return answer_list\n\n# def auto_fill_missing_value(dataframe, column, variable_name, insights, task):\n#     if \"shouldweimpute?:no\" in insights.replace(\" \", \"\").lower():\n#         return dataframe[column], None\n    \n#     system_prompt = f\"You're ml engineer expert\"\n#     instruction_template = \"\"\"\n#     The task:\n#     {task}\n\n#     Your today job is to coding what insights of data analytics gave to you.\n#     {insights}\n\n#     Instruction:\n#         1. Write a pandas code to fix the requirements.\n#         2. Use {df_name} as variable.\n#         3. Do not add quote or backticks.\n#         4. Don't say anything except the python code.\n#         5. Carefully the data type is {data_type} and the column is {column}.\n#         6. Don't change, downcast or editing to output at all cost (data type should be the same).\n#         7. If it category dype please select the most of unique value instead.\n\n#     Note: Remember that DONT CHANGE TYPE AT ALL COST.\n    \n#     Example output\n#     {df_name}.fillna(0, inplace=True)\n#     \"\"\".replace('{df_name}', variable_name).replace(\"{task}\", task).replace(\"{column}\", column)\n#     datatype = str(dataframe[column].dtype)\n#     prompt = instruction_template.replace(\"{insights}\", insights).replace(\"{data_type}\", datatype)\n#     answer = get_answer_from_gemini(system_prompt, prompt)\n#     exec(f\"{variable_name} = dataframe.copy()\")\n#     exec(answer)\n    \n#     return eval(f\"{variable_name}['{column}']\"), answer\n    \n\ndef get_features(column_and_description_pairs, domain, task_description, chunk_size = 10):\n    system_propmpt = f\"You're data analytics expert especially in {domain} domain\"\n    instruction_template = \"\"\"\n    Please consider this below text as a job task description\n    {task_descrition}\n\n    The data columns:\n    {columns_and_description}\n    \n\n    Instruction:\n        - Classify column feature using your domain expert, data analytics skill and provided description to these category (relate, non_relate)\n             class description\n            - relate: The column direct or indirect relate to the task description\n            - non_relate: The column not relate to the task at all\n        - The output classes can be imbalanced.\n        - The output should not have these (backticks, explanation, opinion, quotes) \n        - Answer format {'non_relate': [..., ...], 'relate': [..., ...]}\n    \"\"\".replace(\"{task_descrition}\", task_description)\n\n    groups = {\n        \"relate\": [],\n        \"non_relate\": []\n    }\n\n    for chunk in tqdm(range(0, len(column_and_description_pairs), chunk_size)):\n        column_and_description = column_and_description_pairs[chunk: chunk + chunk_size]\n        text = \"\\n\\n\".join(\n            [f\"Column: {column}\\nDescription: {description}\" for column, description in column_and_description]\n        )\n\n        instruction = instruction_template.replace(\"{columns_and_description}\", text)\n\n        response = get_answer_from_gemini(\n            system_propmpt, \n            instruction, \n            generation_config=stability_generation_config\n        )\n        \n        group = parse_json(response)\n        groups['non_relate'].extend(group['non_relate'])\n        groups['relate'].extend(group['relate'])\n        \n    return groups","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:31:36.909080Z","iopub.execute_input":"2024-05-23T22:31:36.909417Z","iopub.status.idle":"2024-05-23T22:31:36.925531Z","shell.execute_reply.started":"2024-05-23T22:31:36.909392Z","shell.execute_reply":"2024-05-23T22:31:36.924556Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Load columns file","metadata":{}},{"cell_type":"code","source":"# with open(\"/kaggle/input/columns/group_columnv4.json\", 'r') as file:\n#     data = json.load(file)\n    \n# groups = {'relate': [col for col in data['columns'] if col not in [\"case_id\", \"month_decision\", \"weekday_decision\", \"WEEK_NUM\", \"target\"]]}","metadata":{"execution":{"iopub.status.busy":"2024-05-23T18:14:20.699034Z","iopub.execute_input":"2024-05-23T18:14:20.699768Z","iopub.status.idle":"2024-05-23T18:14:20.706660Z","shell.execute_reply.started":"2024-05-23T18:14:20.699733Z","shell.execute_reply":"2024-05-23T18:14:20.705703Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(\"/kaggle/input/home-credit-set/group_column.json\", 'r') as file:\n    groups1 = json.load(file)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:27:52.113996Z","iopub.execute_input":"2024-05-23T22:27:52.114615Z","iopub.status.idle":"2024-05-23T22:27:52.120298Z","shell.execute_reply.started":"2024-05-23T22:27:52.114581Z","shell.execute_reply":"2024-05-23T22:27:52.119344Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(\"/kaggle/input/home-credit-set/group_column_2.json\", 'r') as file:\n    groups2 = json.load(file)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:28:03.750583Z","iopub.execute_input":"2024-05-23T22:28:03.751098Z","iopub.status.idle":"2024-05-23T22:28:03.760029Z","shell.execute_reply.started":"2024-05-23T22:28:03.751053Z","shell.execute_reply":"2024-05-23T22:28:03.759091Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# column_and_description_pairs = list(get_descriptive_columns(valid_columns, features_df))\n# agg_column_and_description_pairs = list(get_descriptive_columns(\n#     [col.replace(\"max_\", \"\") for col in other_columns], features_df))","metadata":{"execution":{"iopub.status.busy":"2024-05-23T18:12:32.254739Z","iopub.execute_input":"2024-05-23T18:12:32.255417Z","iopub.status.idle":"2024-05-23T18:12:32.477824Z","shell.execute_reply.started":"2024-05-23T18:12:32.255384Z","shell.execute_reply":"2024-05-23T18:12:32.476845Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##\n##    SKIP this if you want to load columns\n##\n# chunk_size = 6\n# groups1 = get_features(column_and_description_pairs, domain, task_description, chunk_size = chunk_size)\n# groups2 = get_features(agg_column_and_description_pairs, domain, task_description, chunk_size = chunk_size)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T11:45:26.879815Z","iopub.execute_input":"2024-05-23T11:45:26.880441Z","iopub.status.idle":"2024-05-23T11:48:12.740538Z","shell.execute_reply.started":"2024-05-23T11:45:26.880408Z","shell.execute_reply":"2024-05-23T11:48:12.739634Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#\n#    SKIP this if you want to load columns\n#\nprint(\"Non relate: \", len(groups1['non_relate']) + len(groups2['non_relate']))\nprint(\"Relate: \", len(groups1['relate']) + len(groups2['relate']))","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:28:14.899297Z","iopub.execute_input":"2024-05-23T22:28:14.899677Z","iopub.status.idle":"2024-05-23T22:28:14.905884Z","shell.execute_reply.started":"2024-05-23T22:28:14.899638Z","shell.execute_reply":"2024-05-23T22:28:14.904835Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##\n#    SKIP this if you want to load columns\n#\ngroups = {\n    \"relate\": groups1['relate'] + [\"max_\" + g for g in groups2['relate']],\n    \"non_relate\": groups1['non_relate'] + groups2['non_relate']\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:28:24.857897Z","iopub.execute_input":"2024-05-23T22:28:24.858632Z","iopub.status.idle":"2024-05-23T22:28:24.863282Z","shell.execute_reply.started":"2024-05-23T22:28:24.858599Z","shell.execute_reply":"2024-05-23T22:28:24.862414Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"##\n##    Uncomment this if you want to load columns\n##\n# with open(\"group_columnv4.json\", 'r') as file:\n#     data = json.load(file)\n# groups_new = {\n#     'relate': data['columns']\n# }","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:52:12.100231Z","iopub.execute_input":"2024-05-23T14:52:12.101160Z","iopub.status.idle":"2024-05-23T14:52:12.107096Z","shell.execute_reply.started":"2024-05-23T14:52:12.101115Z","shell.execute_reply":"2024-05-23T14:52:12.106051Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_columns = df_train.columns.tolist()\nrelate_columns = groups['relate'] # or groups_new['relate']\n# non_relate_columns = groups['non_relate']\ntrain_base_column = [\"case_id\", \"month_decision\", \"weekday_decision\", \"WEEK_NUM\", \"target\"]","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:31:06.896469Z","iopub.execute_input":"2024-05-23T22:31:06.896855Z","iopub.status.idle":"2024-05-23T22:31:06.902144Z","shell.execute_reply.started":"2024-05-23T22:31:06.896821Z","shell.execute_reply":"2024-05-23T22:31:06.901133Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(relate_columns)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:31:10.060629Z","iopub.execute_input":"2024-05-23T22:31:10.061001Z","iopub.status.idle":"2024-05-23T22:31:10.066860Z","shell.execute_reply.started":"2024-05-23T22:31:10.060970Z","shell.execute_reply":"2024-05-23T22:31:10.065986Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save columns\n# with open(\"group_columnv3.json\", 'w') as file:\n#     file.write(json.dumps(groups))","metadata":{"execution":{"iopub.status.busy":"2024-05-23T11:48:54.510805Z","iopub.execute_input":"2024-05-23T11:48:54.511698Z","iopub.status.idle":"2024-05-23T11:48:54.516355Z","shell.execute_reply.started":"2024-05-23T11:48:54.511657Z","shell.execute_reply":"2024-05-23T11:48:54.515433Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Validate columns\n# This cell should have no output.\nfor column in relate_columns + train_base_column:\n    if column not in _columns:\n        print(column)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:31:13.221638Z","iopub.execute_input":"2024-05-23T22:31:13.222284Z","iopub.status.idle":"2024-05-23T22:31:13.228670Z","shell.execute_reply.started":"2024-05-23T22:31:13.222249Z","shell.execute_reply":"2024-05-23T22:31:13.227577Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature engineer","metadata":{}},{"cell_type":"code","source":"# print(groups['non_relate'])","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:31:16.467581Z","iopub.execute_input":"2024-05-23T22:31:16.467928Z","iopub.status.idle":"2024-05-23T22:31:16.472235Z","shell.execute_reply.started":"2024-05-23T22:31:16.467900Z","shell.execute_reply":"2024-05-23T22:31:16.471214Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(groups['relate'])","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:31:17.657287Z","iopub.execute_input":"2024-05-23T22:31:17.657648Z","iopub.status.idle":"2024-05-23T22:31:17.663010Z","shell.execute_reply.started":"2024-05-23T22:31:17.657620Z","shell.execute_reply":"2024-05-23T22:31:17.661790Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Feature engineer","metadata":{}},{"cell_type":"code","source":"def get_important_features(column_and_description_pairs, system_propmpt, task_description, chunk_size = 10):\n    instruction_template = \"\"\"\n    Please consider this below text as a job task description\n    {task_descrition}\n\n    The data columns:\n    {columns_and_description}\n\n    Instruction:\n        - Give me an important columns using your knowledge.\n        - If there are no important columns please leave it as an empty string.\n        - Please answer in json format {'importances': [..., ...]}\n    \"\"\".replace(\"{task_descrition}\", task_description)\n    \n    groups = []\n\n    for chunk in tqdm(range(0, len(column_and_description_pairs), chunk_size)):\n        column_and_description = column_and_description_pairs[chunk: chunk + chunk_size]\n        text = \"\\n\\n\".join(\n            [f\"Column: {column}\\nDescription: {description}\" for column, description in column_and_description]\n        )\n        instruction = instruction_template.replace(\"{columns_and_description}\", text)\n\n        response = get_answer_from_gemini(\n            system_propmpt, \n            instruction, \n            generation_config=stability_generation_config\n        )\n        group = parse_json(response)\n        groups.extend(group['importances'])\n    \n    return groups","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:31:20.799804Z","iopub.execute_input":"2024-05-23T22:31:20.800640Z","iopub.status.idle":"2024-05-23T22:31:20.807678Z","shell.execute_reply.started":"2024-05-23T22:31:20.800606Z","shell.execute_reply":"2024-05-23T22:31:20.806738Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"relate_column_and_description_pairs = list(\n    get_descriptive_columns(\n        [col.replace(\"max_\", \"\") for col in groups['relate']], \n        features_df\n    )\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:31:43.276760Z","iopub.execute_input":"2024-05-23T22:31:43.277502Z","iopub.status.idle":"2024-05-23T22:31:43.473702Z","shell.execute_reply.started":"2024-05-23T22:31:43.277469Z","shell.execute_reply":"2024-05-23T22:31:43.472756Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"importances_from_data_analytics = get_important_features(\n    relate_column_and_description_pairs, \n    \"You're data analytics expert.\",\n    task_description,\n    10\n)\n\nimportances_from_domain_expert = get_important_features(\n    relate_column_and_description_pairs, \n    f\"You're {domain} expertise.\",\n    task_description,\n    10\n)\n\nimportances_from_kaggle_master = get_important_features(\n    relate_column_and_description_pairs, \n    f\"ํYou're smartest kaggle competitor in the world\\nYou had solved many kaggle challenges especially in {domain} domain\",\n    task_description,\n    10\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T18:19:43.371973Z","iopub.execute_input":"2024-05-23T18:19:43.372322Z","iopub.status.idle":"2024-05-23T18:25:26.423689Z","shell.execute_reply.started":"2024-05-23T18:19:43.372292Z","shell.execute_reply":"2024-05-23T18:25:26.422722Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(\"/kaggle/input/home-credit-set/importances_from_data_analytics.json\", 'r') as file:\n    importances_from_data_analytics = json.load(file)\n\nwith open(\"/kaggle/input/home-credit-set/importances_from_domain_expert.json\", 'r') as file:\n    importances_from_domain_expert = json.load(file)\n\nwith open(\"/kaggle/input/home-credit-set/importances_from_kaggle_master.json\", 'r') as file:\n    importances_from_kaggle_master = json.load(file)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:32:44.723588Z","iopub.execute_input":"2024-05-23T22:32:44.724408Z","iopub.status.idle":"2024-05-23T22:32:44.741326Z","shell.execute_reply.started":"2024-05-23T22:32:44.724375Z","shell.execute_reply":"2024-05-23T22:32:44.740680Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"From domain expertise\", len(importances_from_domain_expert))\nprint(\"From data analytics\", len(importances_from_data_analytics))\nprint(\"From kaggle master\", len(importances_from_kaggle_master))","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:32:46.475427Z","iopub.execute_input":"2024-05-23T22:32:46.476259Z","iopub.status.idle":"2024-05-23T22:32:46.481304Z","shell.execute_reply.started":"2024-05-23T22:32:46.476229Z","shell.execute_reply":"2024-05-23T22:32:46.480343Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Get importances column for generate new features","metadata":{}},{"cell_type":"code","source":"importances_features = list(set(importances_from_data_analytics).intersection(set(importances_from_domain_expert)).intersection(set(importances_from_kaggle_master)))","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:32:49.054415Z","iopub.execute_input":"2024-05-23T22:32:49.054753Z","iopub.status.idle":"2024-05-23T22:32:49.059657Z","shell.execute_reply.started":"2024-05-23T22:32:49.054727Z","shell.execute_reply":"2024-05-23T22:32:49.058593Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"importances_features_pair = list(get_descriptive_columns(\n    importances_features, \n    features_df)\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:32:51.030682Z","iopub.execute_input":"2024-05-23T22:32:51.031108Z","iopub.status.idle":"2024-05-23T22:32:51.114913Z","shell.execute_reply.started":"2024-05-23T22:32:51.031070Z","shell.execute_reply":"2024-05-23T22:32:51.113817Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Gemini generate ideas","metadata":{}},{"cell_type":"code","source":"import random\n\ndef get_features_engineer_ideas(column_pairs, task_description):\n    system_prompt = \"You're ml engineer expert\"\n    instruction_template = \"\"\"\n    Please consider this below text as a job task description\n    {task_descrition}\n\n    The data columns:\n    {columns_and_description}\n\n    Instruction:\n        - Give me an 1-3 ideas to create new features from provided column.\n        - It would better if you can provide pandas code.\n    \"\"\".replace(\"{task_descrition}\", task_description)\n    \n    text = \"\\n\\n\".join(\n        [f\"Column: {column}\\nDescription: {description}\" for column, description in column_pairs]\n    )\n    prompt = instruction_template.replace('{columns_and_description}', text)\n    answer = get_answer_from_gemini(system_prompt, prompt)\n    return answer","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:32:53.909534Z","iopub.execute_input":"2024-05-23T22:32:53.910398Z","iopub.status.idle":"2024-05-23T22:32:53.916139Z","shell.execute_reply.started":"2024-05-23T22:32:53.910363Z","shell.execute_reply":"2024-05-23T22:32:53.915180Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_pairs = random.choices(importances_features_pair, k=12)\nidea = get_features_engineer_ideas(random_pairs, task_description)\nprint(idea)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T12:45:19.372255Z","iopub.execute_input":"2024-05-23T12:45:19.372611Z","iopub.status.idle":"2024-05-23T12:45:26.431972Z","shell.execute_reply.started":"2024-05-23T12:45:19.372585Z","shell.execute_reply":"2024-05-23T12:45:26.430945Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_pairs = random.choices(importances_features_pair, k=18)\nidea = get_features_engineer_ideas(random_pairs, task_description)\nprint(idea)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T13:01:41.183730Z","iopub.execute_input":"2024-05-23T13:01:41.184655Z","iopub.status.idle":"2024-05-23T13:01:46.320660Z","shell.execute_reply.started":"2024-05-23T13:01:41.184619Z","shell.execute_reply":"2024-05-23T13:01:46.319707Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_pairs = random.choices(importances_features_pair, k=24)\nidea = get_features_engineer_ideas(random_pairs, task_description)\nprint(idea)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T13:06:26.899389Z","iopub.execute_input":"2024-05-23T13:06:26.899732Z","iopub.status.idle":"2024-05-23T13:06:33.017482Z","shell.execute_reply.started":"2024-05-23T13:06:26.899708Z","shell.execute_reply":"2024-05-23T13:06:33.016525Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_pairs = random.choices(importances_features_pair, k=24)\nidea = get_features_engineer_ideas(random_pairs, task_description)\nprint(idea)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T13:13:17.284887Z","iopub.execute_input":"2024-05-23T13:13:17.285730Z","iopub.status.idle":"2024-05-23T13:13:22.219951Z","shell.execute_reply.started":"2024-05-23T13:13:17.285697Z","shell.execute_reply":"2024-05-23T13:13:22.218910Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_pairs = random.choices(importances_features_pair, k=24)\nidea = get_features_engineer_ideas(random_pairs, task_description)\nprint(idea)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T13:14:17.013228Z","iopub.execute_input":"2024-05-23T13:14:17.013613Z","iopub.status.idle":"2024-05-23T13:14:22.523409Z","shell.execute_reply.started":"2024-05-23T13:14:17.013583Z","shell.execute_reply":"2024-05-23T13:14:22.522481Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_pairs = random.choices(importances_features_pair, k=60)\nidea = get_features_engineer_ideas(random_pairs, task_description)\nprint(idea)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T13:33:24.401496Z","iopub.execute_input":"2024-05-23T13:33:24.401942Z","iopub.status.idle":"2024-05-23T13:33:34.690109Z","shell.execute_reply.started":"2024-05-23T13:33:24.401907Z","shell.execute_reply":"2024-05-23T13:33:34.689119Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random_pairs = random.choices(importances_features_pair, k=60)\nidea = get_features_engineer_ideas(random_pairs, task_description)\nprint(idea)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T13:39:28.439021Z","iopub.execute_input":"2024-05-23T13:39:28.440045Z","iopub.status.idle":"2024-05-23T13:39:35.586342Z","shell.execute_reply.started":"2024-05-23T13:39:28.440009Z","shell.execute_reply":"2024-05-23T13:39:35.585216Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_en_pipeline(dataframe):\n    \"\"\"\n    \n        Feature engineering from gemini ideas\n        \n    \"\"\"\n    \n    dataframe['late_payments'] = (dataframe['numinstpaidlate1d_3546852L'] > 0).astype(int)\n    dataframe['has_past_due_instl'] = (dataframe['max_numberofoverdueinstlmaxdat_641D'] > 0).astype(int)\n    dataframe['avg_loan_amount_category'] = pd.cut(dataframe['avglnamtstart24m_4525187A'], bins=[0, 20000, 60000, 150000, 300000], labels=['Low', 'Medium', 'High', 'Very High'])\n    dataframe['credit_card_limit_level'] = pd.cut(dataframe['max_credacc_credlmt_575A'], bins=[0, 10000, 20000, 30000, 40000], labels=[0, 1, 2, 3]).value_counts()\n    dataframe['outgoing_over_main_income'] = (dataframe['max_amtdepositoutgoing_4809442A'] / dataframe['maininc_215A'])\n    dataframe['money_level'] = (dataframe['maininc_215A'] - dataframe['avgpmtlast12m_4525200A'])\n    \n    return dataframe","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:33:02.903859Z","iopub.execute_input":"2024-05-23T22:33:02.904568Z","iopub.status.idle":"2024-05-23T22:33:02.911873Z","shell.execute_reply.started":"2024-05-23T22:33:02.904535Z","shell.execute_reply":"2024-05-23T22:33:02.910874Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"use_columns = groups['relate']\n_columns = df_train.columns\nfor i, column in enumerate(tqdm(use_columns)):\n    if column not in _columns:\n        if \"max_\" + column in _columns:\n            use_columns[i] = \"max_\" + column\n            \nuse_columns = use_columns + train_base_column\nlen(use_columns)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:33:06.709634Z","iopub.execute_input":"2024-05-23T22:33:06.710555Z","iopub.status.idle":"2024-05-23T22:33:06.723859Z","shell.execute_reply.started":"2024-05-23T22:33:06.710519Z","shell.execute_reply":"2024-05-23T22:33:06.722969Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = df_train[use_columns]\ndf_train = df_train.drop(\"WEEK_NUM\", axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:33:10.155083Z","iopub.execute_input":"2024-05-23T22:33:10.155757Z","iopub.status.idle":"2024-05-23T22:33:11.372311Z","shell.execute_reply.started":"2024-05-23T22:33:10.155722Z","shell.execute_reply":"2024-05-23T22:33:11.370918Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:33:21.491216Z","iopub.execute_input":"2024-05-23T22:33:21.491959Z","iopub.status.idle":"2024-05-23T22:33:21.497709Z","shell.execute_reply.started":"2024-05-23T22:33:21.491913Z","shell.execute_reply":"2024-05-23T22:33:21.496811Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Load test","metadata":{}},{"cell_type":"markdown","source":"Please add SPAI dataset","metadata":{}},{"cell_type":"code","source":"TEST_BASE_FILE = Path(\"/kaggle/input/home-credit-credit-risk-modeling/test.parquet\")\nTEST_DIR       = Path(\"/kaggle/input/home-credit-credit-risk-modeling\") / \"test_dataset\" / \"transformed\"","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:33:24.221057Z","iopub.execute_input":"2024-05-23T22:33:24.221413Z","iopub.status.idle":"2024-05-23T22:33:24.226175Z","shell.execute_reply.started":"2024-05-23T22:33:24.221383Z","shell.execute_reply":"2024-05-23T22:33:24.225241Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_store = {\n    \"df_base\": read_file(TEST_BASE_FILE),\n    \"depth_0\": [\n        read_file(TEST_DIR / \"test_static_cb_0.parquet\"),\n        read_files(TEST_DIR / \"test_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TEST_DIR / \"test_applprev_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_a_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_tax_registry_b_1.parquet\", 1),\n#         read_file(TEST_DIR / \"test_tax_registry_c_1.parquet\", 1),\n        read_files(TEST_DIR / \"test_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TEST_DIR / \"test_credit_bureau_b_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_other_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_person_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_deposit_1.parquet\", 1),\n        read_file(TEST_DIR / \"test_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TEST_DIR / \"test_credit_bureau_b_2.parquet\", 2),\n    ]\n}\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:37:52.460073Z","iopub.execute_input":"2024-05-23T22:37:52.460439Z","iopub.status.idle":"2024-05-23T22:37:52.900466Z","shell.execute_reply.started":"2024-05-23T22:37:52.460411Z","shell.execute_reply":"2024-05-23T22:37:52.899677Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in df_test.columns:\n    if i == \"weekday_decision\":\n        print(i)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:36:03.034786Z","iopub.execute_input":"2024-05-23T22:36:03.035646Z","iopub.status.idle":"2024-05-23T22:36:03.040297Z","shell.execute_reply.started":"2024-05-23T22:36:03.035612Z","shell.execute_reply":"2024-05-23T22:36:03.039260Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = feature_eng(**data_store)\n# del data_store\n\n# df_test = df_test.drop([\"case_id\", \"month_decision\", \"weekday_decision\", \"assignmentdate_238D\"])\n# df_test = df_test.drop([\"assignmentdate_4527235D\", \"assignmentdate_4955616D\", \"birthdate_574D\", \"contractssum_5085716L\"])\n# df_test = df_test.drop([\"dateofbirth_337D\", \"dateofbirth_342D\", \"days120_123L\", \"days180_256L\"])","metadata":{"execution":{"iopub.status.busy":"2024-05-23T23:13:14.712794Z","iopub.execute_input":"2024-05-23T23:13:14.713583Z","iopub.status.idle":"2024-05-23T23:13:14.796336Z","shell.execute_reply.started":"2024-05-23T23:13:14.713548Z","shell.execute_reply":"2024-05-23T23:13:14.795542Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:43:04.699802Z","iopub.execute_input":"2024-05-23T22:43:04.700442Z","iopub.status.idle":"2024-05-23T22:43:04.724792Z","shell.execute_reply.started":"2024-05-23T22:43:04.700409Z","shell.execute_reply":"2024-05-23T22:43:04.723966Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"exeption = [\"case_id\", \"target\", \"WEEK_NUM\", \"max_pmtamount_36A\", \"max_processingdate_168D\", \"max_num_group1_12\"]\n\ngc.collect()\ndf_test = df_test.select([col for col in df_train.columns if col not in exeption])\ndf_test, cat_cols = to_pandas(df_test)\ndf_test = reduce_mem_usage(df_test)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-23T23:13:25.038789Z","iopub.execute_input":"2024-05-23T23:13:25.039686Z","iopub.status.idle":"2024-05-23T23:13:25.554625Z","shell.execute_reply.started":"2024-05-23T23:13:25.039652Z","shell.execute_reply":"2024-05-23T23:13:25.553432Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = feature_en_pipeline(df_train) \ndf_test = feature_en_pipeline(df_test)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:44:30.908428Z","iopub.execute_input":"2024-05-23T22:44:30.909258Z","iopub.status.idle":"2024-05-23T22:44:31.208906Z","shell.execute_reply.started":"2024-05-23T22:44:30.909214Z","shell.execute_reply":"2024-05-23T22:44:31.207887Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train=reduce_mem_usage(df_train)\ny = df_train[\"target\"]\n\n\ndf_train= df_train.drop(columns=[\"target\", \"case_id\"])\njoblib.dump((df_train, y, df_test), 'datav7.pkl')","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:44:33.018145Z","iopub.execute_input":"2024-05-23T22:44:33.018992Z","iopub.status.idle":"2024-05-23T22:44:46.325286Z","shell.execute_reply.started":"2024-05-23T22:44:33.018950Z","shell.execute_reply":"2024-05-23T22:44:46.324309Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model training","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_validation, y_train, y_validation = train_test_split(df_train, y, test_size=0.2, random_state=42, stratify=y)\n\nprint(\"X_train shape:\", X_train.shape)\nprint(\"y_train shape:\", y_train.shape)\n\nprint(\"X_validation shape:\", X_validation.shape)\nprint(\"y_validation shape:\", y_validation.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:45:02.323048Z","iopub.execute_input":"2024-05-23T22:45:02.323436Z","iopub.status.idle":"2024-05-23T22:45:07.728223Z","shell.execute_reply.started":"2024-05-23T22:45:02.323407Z","shell.execute_reply":"2024-05-23T22:45:07.727212Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\nimport optuna\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\ndef objective(trial):\n    params = {\n        \"boosting_type\": \"gbdt\",\n        \"colsample_bynode\": trial.suggest_float(\"colsample_bynode\", 0.6, 1.0),\n        \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.6, 1.0),\n        \"device\": \"gpu\",\n        \"extra_trees\": trial.suggest_categorical(\"extra_trees\", [True, False]),\n        \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 0.01, 0.1),\n        \"reg_alpha\": trial.suggest_loguniform(\"reg_alpha\", 0.1, 10.0),\n        \"reg_lambda\": trial.suggest_loguniform(\"reg_lambda\", 1.0, 100.0),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 5, 50),\n        \"n_estimators\": trial.suggest_int(\"n_estimators\", 1000, 3000),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 31, 128),\n        \"objective\": \"binary\",\n        \"random_state\": 42,\n        \"verbose\": -1,\n    }\n\n    model = lgb.LGBMClassifier(**params)\n    \n    fit_params = {\n        \"eval_set\": [(X_validation, y_validation)],\n        \"eval_metric\": \"auc\",\n    }\n\n    model.fit(X_train, y_train, **fit_params)\n    \n    preds = model.predict_proba(X_validation)[:, 1]\n    auc = roc_auc_score(y_validation, preds)\n    return auc\n\nstudy = optuna.create_study(direction=\"maximize\")\nstudy.optimize(objective, n_trials=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:45:19.039641Z","iopub.execute_input":"2024-05-23T22:45:19.040096Z","iopub.status.idle":"2024-05-23T22:56:02.479524Z","shell.execute_reply.started":"2024-05-23T22:45:19.040060Z","shell.execute_reply":"2024-05-23T22:56:02.478608Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_trial = study.best_trial\nprint(\"Best trial parameters:\", best_trial.params)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T22:56:25.822171Z","iopub.execute_input":"2024-05-23T22:56:25.822898Z","iopub.status.idle":"2024-05-23T22:56:25.828053Z","shell.execute_reply.started":"2024-05-23T22:56:25.822868Z","shell.execute_reply":"2024-05-23T22:56:25.827144Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"save_best_params = {\n    'colsample_bynode': 0.7074894680102146, \n    'colsample_bytree': 0.7930020594092116, \n    'extra_trees': False, \n    'learning_rate': 0.07047687268576773, \n    'reg_alpha': 0.7318729618486829, \n    'reg_lambda': 2.0426445540406637, \n    'max_depth': 11, \n    'n_estimators': 2424, \n    'num_leaves': 92,\n    'device': 'gpu'\n}\n\n\nx_best = x = {\n    \"colsample_bynode\": 0.9334153402772278,\n    \"colsample_bytree\": 0.9942281264158934,\n    \"extra_trees\": True,\n    \"learning_rate\": 0.027287553713076496,\n    \"reg_alpha\": 2.0232375669059226,\n    \"reg_lambda\": 47.64369370451841,\n    \"max_depth\": 6,\n    \"n_estimators\": 2552,\n    \"num_leaves\": 64,\n    'device': 'gpu',\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-23T23:00:22.262912Z","iopub.execute_input":"2024-05-23T23:00:22.263795Z","iopub.status.idle":"2024-05-23T23:00:22.271509Z","shell.execute_reply.started":"2024-05-23T23:00:22.263761Z","shell.execute_reply":"2024-05-23T23:00:22.270542Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train the final model with the best parameters\n# best_params = best_trial.params\n# best_params[\"device\"] = \"gpu\"  # Ensure device is set to GPU\n# model = lgb.LGBMClassifier(**best_params)\n\nmodel = lgb.LGBMClassifier(**x_best)\n\n# save_best_params[\"device\"] = \"gpu\"  # Ensure device is set to GPU\nmodel.fit(df_train, y)\n\nfitted_models_lgb = [model]\nprint(\"Model training with Optuna optimization success\")","metadata":{"execution":{"iopub.status.busy":"2024-05-23T23:00:45.927901Z","iopub.execute_input":"2024-05-23T23:00:45.928267Z","iopub.status.idle":"2024-05-23T23:10:03.401016Z","shell.execute_reply.started":"2024-05-23T23:00:45.928238Z","shell.execute_reply":"2024-05-23T23:10:03.400024Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.shape, df_test.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-23T23:11:46.696001Z","iopub.execute_input":"2024-05-23T23:11:46.696357Z","iopub.status.idle":"2024-05-23T23:11:46.702605Z","shell.execute_reply.started":"2024-05-23T23:11:46.696331Z","shell.execute_reply":"2024-05-23T23:11:46.701733Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test","metadata":{"execution":{"iopub.status.busy":"2024-05-23T23:12:08.784759Z","iopub.execute_input":"2024-05-23T23:12:08.785446Z","iopub.status.idle":"2024-05-23T23:12:08.831864Z","shell.execute_reply.started":"2024-05-23T23:12:08.785412Z","shell.execute_reply":"2024-05-23T23:12:08.830909Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(df_test.shape[1]-1):\n    mismatch = df_test.drop(\"case_id\", axis=1).iloc[:, i].dtype == df_train.iloc[:, i].dtype\n    if not mismatch:\n        print(i, df_test.drop(\"case_id\", axis=1).iloc[:, i].dtype, df_train.iloc[:, i].dtype)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T23:11:52.925789Z","iopub.execute_input":"2024-05-23T23:11:52.926646Z","iopub.status.idle":"2024-05-23T23:11:53.713687Z","shell.execute_reply.started":"2024-05-23T23:11:52.926613Z","shell.execute_reply":"2024-05-23T23:11:53.712290Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"case_id = df_test['case_id']","metadata":{"execution":{"iopub.status.busy":"2024-05-23T23:12:26.340361Z","iopub.execute_input":"2024-05-23T23:12:26.340718Z","iopub.status.idle":"2024-05-23T23:12:26.575265Z","shell.execute_reply.started":"2024-05-23T23:12:26.340690Z","shell.execute_reply":"2024-05-23T23:12:26.574051Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":" def to_categorical(df_train): \n    for i in df_train.columns:\n        if df_train[i].dtype.name == \"category\":\n            df_train[i] = df_train[i].cat.remove_unused_categories()\n\n    return df_train\n\ndf_train = to_categorical(df_train)\ndf_test = to_categorical(df_test).drop('case_id', axis=1)\n\nfor i in df_train.columns:\n    df_test[i] = df_test[i].astype(df_train[i].dtype)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:11:45.199584Z","iopub.execute_input":"2024-05-23T14:11:45.199970Z","iopub.status.idle":"2024-05-23T14:11:47.782245Z","shell.execute_reply.started":"2024-05-23T14:11:45.199939Z","shell.execute_reply":"2024-05-23T14:11:47.781411Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Prediction","metadata":{}},{"cell_type":"code","source":"prediction = model.predict_proba(df_test)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:11:52.220934Z","iopub.execute_input":"2024-05-23T14:11:52.221710Z","iopub.status.idle":"2024-05-23T14:11:56.076923Z","shell.execute_reply.started":"2024-05-23T14:11:52.221679Z","shell.execute_reply":"2024-05-23T14:11:56.076038Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"prediction[:, 1].min()","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:12:05.517984Z","iopub.execute_input":"2024-05-23T14:12:05.518683Z","iopub.status.idle":"2024-05-23T14:12:05.525345Z","shell.execute_reply.started":"2024-05-23T14:12:05.518632Z","shell.execute_reply":"2024-05-23T14:12:05.524300Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test['case_id'] = case_id","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:12:08.621567Z","iopub.execute_input":"2024-05-23T14:12:08.621980Z","iopub.status.idle":"2024-05-23T14:12:08.628631Z","shell.execute_reply.started":"2024-05-23T14:12:08.621930Z","shell.execute_reply":"2024-05-23T14:12:08.627340Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/home-credit-credit-risk-modeling/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:12:10.283177Z","iopub.execute_input":"2024-05-23T14:12:10.284070Z","iopub.status.idle":"2024-05-23T14:12:10.308111Z","shell.execute_reply.started":"2024-05-23T14:12:10.284039Z","shell.execute_reply":"2024-05-23T14:12:10.307319Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"answer_list = []\nfor i, (idx, row) in enumerate(tqdm(submission.iterrows())):\n    index = df_test[df_test['case_id'] == int(row['case_id'])].index[0]\n    answer_list.append(prediction[index, 1])","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:12:14.532584Z","iopub.execute_input":"2024-05-23T14:12:14.533314Z","iopub.status.idle":"2024-05-23T14:15:11.077683Z","shell.execute_reply.started":"2024-05-23T14:12:14.533277Z","shell.execute_reply":"2024-05-23T14:15:11.076577Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission['target'] = answer_list #.iloc[5:] = answer_list[5:]","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:15:13.291333Z","iopub.execute_input":"2024-05-23T14:15:13.292079Z","iopub.status.idle":"2024-05-23T14:15:13.302903Z","shell.execute_reply.started":"2024-05-23T14:15:13.292046Z","shell.execute_reply":"2024-05-23T14:15:13.301793Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv(\"Feeling so high but too far away to hold me\" + \".csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:15:15.792492Z","iopub.execute_input":"2024-05-23T14:15:15.793288Z","iopub.status.idle":"2024-05-23T14:15:15.862573Z","shell.execute_reply.started":"2024-05-23T14:15:15.793235Z","shell.execute_reply":"2024-05-23T14:15:15.861718Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import joblib\n# save model\njoblib.dump(model, 'lgb.pkl')\n# load model\n# gbm_pickle = joblib.load('lgb.pkl')","metadata":{"execution":{"iopub.status.busy":"2024-05-23T15:50:42.913796Z","iopub.execute_input":"2024-05-23T15:50:42.914831Z","iopub.status.idle":"2024-05-23T15:50:43.519367Z","shell.execute_reply.started":"2024-05-23T15:50:42.914785Z","shell.execute_reply":"2024-05-23T15:50:43.518324Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb.plot_importance(model, importance_type=\"gain\", figsize=(7, 40), title=\"LightGBM Feature Importance (Gain)\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:19:29.544147Z","iopub.execute_input":"2024-05-23T14:19:29.544589Z","iopub.status.idle":"2024-05-23T14:19:34.341868Z","shell.execute_reply.started":"2024-05-23T14:19:29.544555Z","shell.execute_reply":"2024-05-23T14:19:34.340709Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:36:50.627239Z","iopub.execute_input":"2024-05-23T14:36:50.628248Z","iopub.status.idle":"2024-05-23T14:36:50.634656Z","shell.execute_reply.started":"2024-05-23T14:36:50.628211Z","shell.execute_reply":"2024-05-23T14:36:50.633629Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"set(df_train.columns) - set(use_columns)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:37:19.130719Z","iopub.execute_input":"2024-05-23T14:37:19.131139Z","iopub.status.idle":"2024-05-23T14:37:19.138438Z","shell.execute_reply.started":"2024-05-23T14:37:19.131103Z","shell.execute_reply":"2024-05-23T14:37:19.137329Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"set(use_columns) - set(df_train.columns)","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:37:47.522102Z","iopub.execute_input":"2024-05-23T14:37:47.523060Z","iopub.status.idle":"2024-05-23T14:37:47.529992Z","shell.execute_reply.started":"2024-05-23T14:37:47.523024Z","shell.execute_reply":"2024-05-23T14:37:47.528917Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(\"group_columnv4.json\", 'w') as file:\n    file.write(json.dumps({\"columns\": use_columns}))","metadata":{"execution":{"iopub.status.busy":"2024-05-23T14:38:04.531852Z","iopub.execute_input":"2024-05-23T14:38:04.532291Z","iopub.status.idle":"2024-05-23T14:38:04.538446Z","shell.execute_reply.started":"2024-05-23T14:38:04.532244Z","shell.execute_reply":"2024-05-23T14:38:04.537345Z"},"trusted":true},"outputs":[],"execution_count":null}]}