{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":6825799,"sourceType":"datasetVersion","datasetId":3922922},{"sourceId":8517318,"sourceType":"datasetVersion","datasetId":4992448}],"dockerImageVersionId":30648,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt\nimport seaborn as sn\nimport joblib\npd.set_option('display.max_colwidth', 150)\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n! pip install /kaggle/input/duckdb/duckdb-0.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\nimport duckdb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/home-credit-credit-risk-model-stability/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_base(data_type):\n    data_path = path + f'parquet_files/{data_type}/'\n    t = pd.read_parquet(data_path + f'{data_type}_base.parquet')\n    t['date_decision'] = pd.to_datetime(t['date_decision'])\n    t['MONTH'] = t['date_decision'].to_numpy().astype('datetime64[M]')\n    \n    return t","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(t, data_type, train_mask, dic, dic_2):\n    not_features = ['case_id', 'date_decision', 'MONTH', 'WEEK_NUM', 'target']\n    date_cols = [col for col in t.columns if col[-1] == 'D' and col not in not_features]\n    not_features = not_features + date_cols\n    num_cols = [col for col in t.iloc[:1].select_dtypes(include=np.number).columns.tolist() if col not in not_features]\n    cat_cols = [col for col in t.iloc[:1].select_dtypes(exclude=np.number).columns.tolist() if col not in not_features]\n    not_features = not_features + cat_cols\n    features = [col for col in t.columns if col not in not_features]\n    \n    for col in date_cols:\n        t[col] = pd.to_datetime(t[col])\n        t[col] = (t['date_decision'] - t[col]).dt.days\n       \n    if data_type == 'train':\n        dic = {}\n        dic_2 = {}\n        for col in cat_cols:\n            s = t[col][train_mask].value_counts(normalize=True, dropna=False) > 0.01\n            dic[col] = s[s].index.to_list()\n\n    for col in dic.keys():\n        t[col] = t[col].map(lambda x: x if x in dic[col] else np.nan).astype('category')\n\n    for col in dic.keys():\n        if data_type == 'train':\n            f = t[[col, 'target']]\n            f = f[train_mask]\n            events = pd.DataFrame(f.groupby(col, dropna=False)['target'].sum()).rename(columns={'target': 'event'})\n            events['N'] = f.groupby(col, dropna=False)['target'].count()\n            events['non_event'] = events['N'] - events['event']\n            events['pct_event'] = events['event'] / events['event'].sum()\n            events['pct_non_event'] = events['non_event'] / events['non_event'].sum()\n            events['pct_N'] = events['N'] / events['N'].sum()\n            events['woe'] = np.log(events['pct_non_event'] / events['pct_event'])\n            events['dr'] = f.groupby(col, dropna=False)['target'].mean()\n            #events.loc[events['pct_N'] < 0.01, 'woe'] = 0\n            if np.nan in events.index:\n                events = events.drop(np.nan)\n            dic_2[col] = events\n\n        t[col] = t[col].map(dic_2[col]['woe']).astype(np.float64)#.fillna(0)\n        \n    return t, dic, dic_2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def woe_missing(t, train_mask, mask, col):\n    _, bins = pd.qcut(t[col][train_mask], q=5, duplicates='drop', retbins=True)\n    bins = np.insert(bins, 0, -np.inf)\n    bins = np.append(bins, np.inf)\n\n    f = t[[col, 'target']]\n    f['bin'] = pd.cut(t[col], bins)\n    f = f[mask]\n    events = pd.DataFrame(f.groupby('bin', dropna=False)['target'].sum()).rename(columns={'target': 'event'})\n    events['non_event'] = f.groupby('bin', dropna=False)['target'].count() - events['event']\n    events['pct_event'] = events['event'] / events['event'].sum()\n    events['pct_non_event'] = events['non_event'] / events['non_event'].sum()\n    events['woe'] = np.log(events['pct_non_event'] / events['pct_event'])\n    events['dr'] = f.groupby('bin', dropna=False)['target'].mean()\n    \n    if None in events.index:\n        return events.loc[None]['woe']\n    return np.nan","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def psi(col, t, train_mask, valid_mask, use_miss):\n    _, bins = pd.qcut(t[col][train_mask], q=5, duplicates='drop', retbins=True)\n    bins = np.insert(bins, 0, -np.inf)\n    bins = np.append(bins, np.inf)\n\n    g = pd.cut(t[col], bins)\n\n    expected = g[train_mask].value_counts(normalize = True, dropna = 1 - use_miss)\n    actual = g[valid_mask].value_counts(normalize = True, dropna = 1 - use_miss)\n\n    h = pd.DataFrame(expected).rename(columns={'proportion': 'expected'})\n    h['actual'] = actual\n    h['actual'] = np.clip(h['actual'].fillna(0), 0.001, 0.999)\n    h['expected'] = np.clip(h['expected'], 0.001, 0.999)\n\n    return np.sum((h['expected'] - h['actual']) * np.log(h['expected'] / h['actual']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gini(t, mask, col):\n    f = t[[col, 'target']]\n    f = f[mask]\n    f = f[f[col].notnull()]\n    return 2*roc_auc_score(f['target'], f[col]) - 1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_table(t, data_type, dic, dic_2, res, keyword, aggr_types, descr):\n    if data_type == 'train':\n        train_mask = (t['WEEK_NUM'] < 65)\n        valid_mask = (t['WEEK_NUM'] >= 65)\n    if data_type == 'train':\n        t, dic, dic_2 = process(t, 'train', train_mask, None, None)\n    else:\n        t, _, _2 = process(t, 'test', None, dic, dic_2)\n    \n    if data_type == 'train':    \n        for col in t.columns:\n            if col not in descr.index:\n                descr.loc[col] = [np.nan for i in range(descr.shape[1])]\n                \n            descr.loc[col, 'missing_part'] = t[col].isnull().mean()\n            descr.loc[col, 'top_part'] = t[col].value_counts(normalize=True, dropna=False).iloc[0]\n            try: descr.loc[col, 'psi'] = psi(col, t, train_mask, valid_mask, True)\n            except: pass\n            try: descr.loc[col, 'psi_non_miss'] = psi(col, t, train_mask, valid_mask, False)\n            except: pass\n            try: descr.loc[col, 'miss_woe_train'] = woe_missing(t, train_mask, train_mask, col)\n            except: pass\n            try: descr.loc[col, 'miss_woe_valid'] = woe_missing(t, train_mask, valid_mask, col)\n            except: pass\n            try: descr.loc[col, 'gini_train'] = gini(t, train_mask, col)\n            except: pass\n            try: descr.loc[col, 'gini_valid'] = gini(t, valid_mask, col)\n            except: pass\n        descr['miss_woe_delta'] = descr['miss_woe_train'] - descr['miss_woe_valid']\n        descr['gini_change_sign'] = (descr['gini_train'] * descr['gini_valid'] < 0)\n        descr['gini_train'] = descr['gini_train'].abs()\n        descr['gini_valid'] = descr['gini_valid'].abs()\n        \n        not_features = ['case_id', 'date_decision', 'MONTH', 'WEEK_NUM', 'target', 'num_group1', 'num_group2']\n        date_cols = [col for col in t.columns if col[-1] == 'D' and col not in not_features]\n        features = [col for col in t.columns if col not in not_features]\n        res = descr.loc[features].loc[(descr['gini_valid'] >= 0.04) & (descr['gini_train'] >= 0.02) & \n                                (descr['gini_valid'] >= 0.8*descr['gini_train']) & \n                                (descr['gini_change_sign'] == False)].index.to_list()\n    res = [col for col in res if len(dic.get(col, [])) < 7]\n        \n    if aggr_types == None:\n        out = t[res + ['case_id']].set_index('case_id').rename(columns=lambda col: f'{keyword}_' + col)\n        if data_type == 'train':\n            return out, dic, dic_2, res, descr\n        return out, None, None, None, None\n    \n    out = t[res + ['case_id']].groupby('case_id').max().rename(columns=lambda col: f'{keyword}_max_' + col)\n    out_2 = t[res + ['case_id']].groupby('case_id').min().rename(columns=lambda col: f'{keyword}_min_' + col)\n    out_3 = t[res + ['case_id']].groupby('case_id').mean().rename(columns=lambda col: f'{keyword}_avg_' + col)\n    if 'num_group2' not in t.columns:\n        out_4 = t[(t['num_group1'] == 0)]\n    else:\n        out_4 = t[(t['num_group1'] == 0) & (t['num_group2'] == 0)]\n    out_4 = out_4[res + ['case_id']].set_index('case_id').rename(columns=lambda col: f'{keyword}_last_' + col)\n    out_5 = pd.DataFrame(t[['case_id']].groupby('case_id').size()).rename(columns={0: f'{keyword}_cnt'})\n    output = {'max': out, 'min': out_2, 'avg': out_3, 'last': out_4, 'cnt': out_5}\n    if data_type == 'train':\n        return pd.concat([output[i] for i in aggr_types], axis = 1), dic, dic_2, res, descr\n    return pd.concat([output[i] for i in aggr_types], axis = 1), None, None, None, None","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_table(t, data_type, s, key, k_train, k_test, table_name, aggr_types, descr, pre_compute):\n    data_path = path + f'parquet_files/{data_type}/'\n    k = k_test if data_type == 'test' else k_train\n    if data_type == 'train' and pre_compute == False:\n        s[key] = {}\n        f = t[['case_id', 'target', 'WEEK_NUM', 'date_decision']].set_index('case_id')\n        if k == 1:\n            df = pd.read_parquet(data_path + f'{data_type}_{table_name}.parquet').join(f, on='case_id', how='left')\n        else:\n            df = pd.concat([pd.read_parquet(data_path + f'{data_type}_{table_name}_{i}.parquet').join(f, on='case_id', how='left') for i in range(k)],\n                         ignore_index = True, axis = 0)\n        gc.collect()\n        s[key]['cols'] = df.columns\n        df, s[key]['dic'], s[key]['dic_2'], s[key]['res'], descr = process_table(df, 'train', None, None, None, key, aggr_types, descr)\n    else:\n        f = t[['case_id', 'date_decision']].set_index('case_id')\n        if k == 1:\n            df = pd.read_parquet(data_path + f'{data_type}_{table_name}.parquet').join(f, on='case_id', how='left')\n        else:\n            df = pd.concat([pd.read_parquet(data_path + f'{data_type}_{table_name}_{i}.parquet').join(f, on='case_id', how='left') for i in range(k)],\n                         ignore_index = True, axis = 0)\n        gc.collect()\n        df, _, _2, _3, _4 = process_table(df, 'test', s[key]['dic'], s[key]['dic_2'], s[key]['res'], key, aggr_types, descr)\n\n    t = t.join(df, on = 'case_id', how = 'left')\n    df = pd.DataFrame()\n    gc.collect()\n    \n    return t, s, descr","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_type = 'train'\nt = load_base(data_type)\npre_compute = True\nif pre_compute == False:\n    descr = pd.read_csv(path + 'feature_definitions.csv')\n    descr = descr.set_index('Variable')\n    s = {}\nelse:\n    s = joblib.load('/kaggle/input/hc2024-test/s.pkl')\n    descr = joblib.load('/kaggle/input/hc2024-test/descr.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\nt, s, descr = load_table(t, data_type, s, 'static', 2, 3, 'static_0', None, descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'applprev', 2, 3, 'applprev_1', ['avg', 'cnt', 'max', 'min'], descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'person', 1, 1, 'person_1', ['last', 'cnt', 'max', 'min'], descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'cb_a', 4, 5, 'credit_bureau_a_1', ['avg', 'cnt', 'max', 'min'], descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'cb_static', 1, 1, 'static_cb_0', None, descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'cb_b', 1, 1, 'credit_bureau_b_1', ['avg', 'cnt', 'max', 'min'], descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'tax_a', 1, 1, 'tax_registry_a_1', ['avg', 'cnt', 'max', 'min'], descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'tax_b', 1, 1, 'tax_registry_b_1', ['avg', 'cnt'], descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'tax_c', 1, 1, 'tax_registry_c_1', ['avg', 'cnt'], descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'other', 1, 1, 'other_1', ['last'], descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'deposit', 1, 1, 'deposit_1', ['max', 'cnt'], descr, pre_compute)\nt, s, descr = load_table(t, data_type, s, 'debitcard', 1, 1, 'debitcard_1', ['max', 'cnt'], descr, pre_compute)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_type = 'test'\ntest = load_base(data_type)\npre_compute = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\ntest, s, descr = load_table(test, data_type, s, 'static', 2, 3, 'static_0', None, descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'applprev', 2, 3, 'applprev_1', ['avg', 'cnt', 'max', 'min'], descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'person', 1, 1, 'person_1', ['last', 'cnt', 'max', 'min'], descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'cb_a', 4, 5, 'credit_bureau_a_1', ['avg', 'cnt', 'max', 'min'], descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'cb_static', 1, 1, 'static_cb_0', None, descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'cb_b', 1, 1, 'credit_bureau_b_1', ['avg', 'cnt', 'max', 'min'], descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'tax_a', 1, 1, 'tax_registry_a_1', ['avg', 'cnt', 'max', 'min'], descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'tax_b', 1, 1, 'tax_registry_b_1', ['avg', 'cnt'], descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'tax_c', 1, 1, 'tax_registry_c_1', ['avg', 'cnt'], descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'other', 1, 1, 'other_1', ['last'], descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'deposit', 1, 1, 'deposit_1', ['max', 'cnt'], descr, pre_compute)\ntest, s, descr = load_table(test, data_type, s, 'debitcard', 1, 1, 'debitcard_1', ['max', 'cnt'], descr, pre_compute)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def psi_test(col, t, train_mask, test, use_miss):\n    _, bins = pd.qcut(t[col][train_mask], q=5, duplicates='drop', retbins=True)\n    bins = np.insert(bins, 0, -np.inf)\n    bins = np.append(bins, np.inf)\n\n    g = pd.cut(t[col], bins)\n    f = pd.cut(test[col], bins)\n\n    expected = g[train_mask].value_counts(normalize = True, dropna = 1 - use_miss)\n    actual = f.value_counts(normalize = True, dropna = 1 - use_miss)\n\n    h = pd.DataFrame(expected).rename(columns={'proportion': 'expected'})\n    h['actual'] = actual\n    h['actual'] = np.clip(h['actual'].fillna(0), 0.001, 0.999)\n    h['expected'] = np.clip(h['expected'], 0.001, 0.999)\n\n    return np.sum((h['expected'] - h['actual']) * np.log(h['expected'] / h['actual']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_mask = (t['WEEK_NUM'] < 95)\nvalid_mask = (t['WEEK_NUM'] >= 65)\noot_mask = (t['WEEK_NUM'] >= 80)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\nfor col in t.columns:\n    #descr.loc[col, 'missing_part'] = t[col].isnull().mean()\n    #descr.loc[col, 'miss_part_train'] = t[col][train_mask].isnull().mean()\n    #descr.loc[col, 'miss_part_valid'] = t[col][valid_mask].isnull().mean()\n    #descr.loc[col, 'top_part'] = t[col].value_counts(normalize=True, dropna=False).iloc[0]\n    #try: descr.loc[col, 'psi'] = psi(col, t, train_mask, valid_mask, True)\n    #except: pass\n    #try: descr.loc[col, 'psi_non_miss'] = psi(col, t, train_mask, valid_mask, False)\n    #except: pass\n    try: descr.loc[col, 'psi_test'] = psi_test(col, t, ~valid_mask, test, True)\n    except: pass\n    try: descr.loc[col, 'psi_non_miss_test'] = psi_test(col, t, ~valid_mask, test, False)\n    except: pass\n    #try: descr.loc[col, 'miss_woe_train'] = woe_missing(t, train_mask, train_mask, col)\n    #except: pass\n    #try: descr.loc[col, 'miss_woe_valid'] = woe_missing(t, train_mask, valid_mask, col)\n    #except: pass\n    #try: descr.loc[col, 'gini_train'] = gini(t, train_mask, col)\n    #except: pass\n    #try: descr.loc[col, 'gini_valid'] = gini(t, valid_mask, col)\n    #except: pass\n#descr['miss_woe_delta'] = descr['miss_woe_train'] - descr['miss_woe_valid']\n#descr['gini_change_sign'] = (descr['gini_train'] * descr['gini_valid'] < 0)\n#descr['gini_train'] = descr['gini_train'].abs()\n#descr['gini_valid'] = descr['gini_valid'].abs()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if pre_compute == False:\n    joblib.dump(s, 's.pkl')\n    joblib.dump(descr, 'descr.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"not_features = ['case_id', 'date_decision', 'MONTH', 'WEEK_NUM', 'target']\ndate_cols = [col for col in t.columns if col[-1] == 'D' and col not in not_features]\nnum_cols = [col for col in t.select_dtypes(include=np.number).columns.tolist() if col not in not_features]\ncat_cols = [col for col in t.select_dtypes(exclude=np.number).columns.tolist() if col not in not_features]\nnot_features = not_features + cat_cols\nfeatures = descr.loc[(descr['gini_valid'] >= 0.04) & (descr['gini_train'] >= 0.02) & \n                                (descr['gini_valid'] >= 0.8*descr['gini_train']) & \n                                (descr['gini_change_sign'] == False) & \n                                (descr['psi_non_miss'] < 0.9) &\n                                (descr['psi_non_miss_test'] < 0.15)].sort_values(by = 'gini_valid').index.to_list()\nfeatures = [col for col in features if col not in not_features]\nfeatures = [col for col in features if col in t.columns]\nlen(features)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = t['target']\nX_train = t.loc[train_mask, features]\ny_train = y.loc[train_mask]\nX_valid = t.loc[valid_mask, features]\ny_valid = y.loc[valid_mask]\nX_oot = t.loc[oot_mask, features]\ny_oot = y.loc[oot_mask]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\n\nparams = {\n    'n_estimators'     : 5000,\n    'objective'        : 'binary',\n    'metric'           : 'auc',\n    'verbose'          : -1,\n    'learning_rate'    : 0.05, \n    'colsample_bytree' : 0.5099540080762465, \n    'colsample_bynode' : 0.4575886155554022, \n    'lambda_l1'        : 3.2150774310219994, \n    'lambda_l2'        : 6.026390839987788, \n    'min_data_in_leaf' : 86, \n    'max_depth'        : 5, \n    'num_leaves'       : 174, \n    'max_bin'          : 255,\n    'device'           : 'gpu',\n    'early_stopping_rounds': 500\n}\n\nmodel = lgb.LGBMClassifier(**params, random_state=100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\n\nparams_4 = {\n    'reg_lambda': 2.954282453141972,\n    'reg_alpha': 2.714226004554789,\n    'subsample': 0.8747340624732572,\n    'colsample_bytree': 0.28932012021539283,\n    'gamma': 2.2132144067786714,\n    'min_child_weight': 100,\n    'max_depth': 4,\n    'learning_rate': 0.04,\n    'tree_method': 'hist',\n    'n_estimators': 8000,\n    #'early_stopping_rounds': 500,\n    'device': 'cuda'\n}\n\nmodel = XGBClassifier(**params_4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], verbose=False)\nif params_4.get('early_stopping_rounds') is not None:\n    print(f\"Training until validation scores don't improve for {params_4.get('early_stopping_rounds')} rounds\")\n    print('Early stopping, best iteration is:')\n    print(f\"[{model.best_iteration}]    valid_0's binary_logloss: {model.best_score}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\np_train = model.predict_proba(X_train)[:, 1]\np_valid = model.predict_proba(X_valid)[:, 1]\np_oot = model.predict_proba(X_oot)[:, 1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(roc_auc_score(y_train, p_train))\nprint(roc_auc_score(y_valid, p_valid))\nprint(roc_auc_score(y_oot, p_oot))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def objective(trial):\n        \n    params_op = {\n    'reg_lambda': trial.suggest_float('reg_lambda', 0.0, 10.0),\n    'reg_alpha': trial.suggest_float('reg_alpha', 0.0, 10.0),\n    'subsample': trial.suggest_float('subsample', 0.4, 1.0),\n    'colsample_bytree': trial.suggest_float('colsample_bytree', 0.2, 1.0),\n    'gamma': trial.suggest_float('gamma', 0.2, 3.0),\n    'min_child_weight': trial.suggest_int('min_child_weight', 20, 1000),\n    'max_depth': trial.suggest_int('max_depth', 3, 10),\n    'learning_rate': trial.suggest_float('learning_rate', 0.004, 0.05),\n    'tree_method': 'hist',\n    'n_estimators': 7000,\n    'early_stopping_rounds': 500,\n    'device': 'cuda'\n}\n\n    model_op = XGBClassifier(**params_op)\n    model_op.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], verbose=False)\n    \n    p_valid = model_op.predict_proba(X_valid)[:, 1]\n    \n\n    return roc_auc_score(y_valid, p_valid)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import optuna\n#study = optuna.create_study(direction='maximize', study_name='Classifier')\n#study.optimize(objective, n_trials=80, show_progress_bar=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fig = optuna.visualization.plot_parallel_coordinate(study)\n#fig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#study.best_params","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = pd.DataFrame()\nX_train = pd.DataFrame()\nX_valid = pd.DataFrame()\nX_oot = pd.DataFrame()\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = test.loc[:, features]\ntest['score'] = np.array(model.predict_proba(X)[:, 1], dtype=np.float64)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm = test[['case_id', 'score']].copy()\nsubm['case_id'] = subm['case_id'].to_numpy()\nsubm = subm.set_index('case_id')\nsubm.to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subm","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}