{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a href=\"https://colab.research.google.com/github/gauravbrills/kaggle-fiddle/blob/main/amex/amex_catboost.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>","metadata":{"id":"view-in-github"}},{"cell_type":"markdown","source":"### CATBOOST AMEX","metadata":{"id":"6S1tGBZ5grxs"}},{"cell_type":"markdown","source":"### Pre-requisites","metadata":{"id":"YsemIH1bhEV-"}},{"cell_type":"markdown","source":"### Imports and Constants","metadata":{"id":"_RMauooTHaLw"}},{"cell_type":"code","source":"import os,random \nimport tqdm \nimport pandas as cudf\nimport numpy as cupy \nfrom catboost import CatBoostClassifier\nimport numpy as np\nfrom sklearn.preprocessing import PolynomialFeatures\nimport joblib\nimport pathlib\nimport tqdm\nfrom sklearn.model_selection import StratifiedKFold\ncudf.set_option('display.max_rows', 500)\ncudf.set_option('display.max_columns', 500)\ncudf.set_option('display.width', 1000)\n\nclass CFG:\n  seed = 42\n  INPUT = \"../input\"\n  TRAIN = True\n  OPTIMIZE = False\n  INFER = False\n  n_folds = 5\n  target ='target'\n  DEBUG= False \n  ADD_CAT = True\n  ADD_LAG = True \n  COMPUTE_Z = True\n  ADD_DIFF_1 = True\n  ADD_DIFF =  [1,3,5]#[3,6]\n  ADD_PCTDIFF = [1,3,6]\n  KURT = False\n  TRIM=True   \n  model_dir = \"\"\n\npath = f'{CFG.INPUT}/amex-data-integer-dtypes-parquet-format'   \n# ====================================================\n# Seed everything\n# ====================================================\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nseed_everything(CFG.seed)  ","metadata":{"id":"nD6axCLvgrx3","execution":{"iopub.status.busy":"2022-07-27T00:31:38.066564Z","iopub.execute_input":"2022-07-27T00:31:38.067134Z","iopub.status.idle":"2022-07-27T00:31:39.175655Z","shell.execute_reply.started":"2022-07-27T00:31:38.067015Z","shell.execute_reply":"2022-07-27T00:31:39.174696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Libs 4 Feature Engg","metadata":{"id":"QM2Lb6RdY5-h"}},{"cell_type":"markdown","source":"### Feature Engineering","metadata":{"id":"SWPQupqBgrx4"}},{"cell_type":"markdown","source":"#### Utils","metadata":{"id":"svO8--Jl3LHJ"}},{"cell_type":"code","source":"def agg_df_num(df):\n    df_agg = df.groupby('customer_ID').agg(f_names)\n    df_agg.columns = [str(c[0])+'_'+str(c[1]) for c in df_agg.columns]\n    return df_agg\n\n# ====================================================\n# Get the difference  --> capture fluctuations, can capture diff(1),diff(2),diff(3) and consider adding features\n# ====================================================\ndef get_difference(data, num_features,period=1): \n    df1 = []\n    customer_ids = []\n    for customer_id, df in  data.groupby(['customer_ID']):\n        # Get the differences\n        diff_df1 = df[num_features].diff(period).iloc[[-1]].values.astype(np.float32)\n        # Append to lists\n        df1.append(diff_df1)\n        customer_ids.append(customer_id)\n    # Concatenate\n    df1 = np.concatenate(df1, axis = 0)\n    # Transform to dataframe\n    df1 = pd.DataFrame(df1, columns = [col + f'_diff{period}' for col in df[num_features].columns])\n    # Add customer id\n    df1['customer_ID'] = customer_ids\n    return df1\n\ndef get_pct_change(data, num_features,period=1): \n    df1 = []\n    customer_ids = []\n    for customer_id, df in  data.groupby(['customer_ID']):\n        # Get the differences\n        diff_df1 = df[num_features].pct_change(period,fill_method=None).iloc[[-1]].values.astype(np.float32)\n        # Append to lists\n        df1.append(diff_df1)\n        customer_ids.append(customer_id)\n    # Concatenate\n    df1 = np.concatenate(df1, axis = 0)\n    # Transform to dataframe\n    df1 = pd.DataFrame(df1, columns = [col + f'_pct_chg{period}' for col in df[num_features].columns])\n    # Add customer id\n    df1['customer_ID'] = customer_ids\n    return df1\n\n\ndef kurtosis(x):\n    if not isinstance(x, pd.Series):\n        x = pd.Series(x)\n    return pd.Series.kurtosis(x)    \n \n\nCID =\"customer_ID\"\nTIME = \"S_2\"\nTARGET = \"target\"\ndef pivot_data(df, train=True):\n    cols = [c for c in df.columns if c not in [CID, TIME, TARGET]]\n    tmp = df.copy()\n    tmp['max'] = tmp.groupby([CID])[TIME].transform('max')\n    tmp['size'] = tmp.groupby([CID])[TIME].transform('size')\n    tmp['rank'] = tmp.groupby([CID])[TIME].transform('rank')\n    tmp['statement'] = (tmp['size']-tmp['rank']).astype(np.int8)\n    pivot_pd = tmp.pivot(index=CID,columns=['statement'],values=cols)\n    pivot_pd.columns = [('{0}__TE{1}'.format(*tup)) for tup in pivot_pd.columns]\n    pivot_pd = pivot_pd.reset_index()\n    return pivot_pd\n\ndef agg_pct_rank_by_cat(df,main_features_last,cat_features_last):\n    df_list = [] \n    for c in cat_features_last:\n        df_agg = df[main_features_last].groupby(df[c]).transform('rank')/df[main_features_last].groupby(df[c]).transform('count')\n        df_agg.columns = [f+'_pct_rank_by_'+c for f in df_agg.columns]\n        df_list.append(df_agg.astype('float16')) \n    return pd.concat([df,pd.concat(df_list,axis=1).astype('float16')], axis=1)\ndef agg_global_rank(df,main_features_last):\n    df_rank = df[main_features_last].transform('rank')\n    df_rank.columns = [s+'_global_rank' for s in df_rank.columns]\n    return pd.concat([df,(df_rank/len(df)).astype('float16')],axis=1)\n\ndef agg_standardize_by_cat(df,main_features_last,cat_features_last):\n    df_list = []\n    for c in cat_features_last:\n        df_agg = df[main_features_last].groupby(df[c]).transform(lambda x: (x - x.mean()) / x.std())\n        df_agg.columns = [f+'_standardized_by_'+c for f in df_agg.columns]\n        df_list.append(df_agg.astype('float16'))\n\n    return pd.concat([df,pd.concat(df_list,axis=1).astype('float16')],axis=1)","metadata":{"id":"a8gnTxoeYN_K","execution":{"iopub.status.busy":"2022-07-27T00:31:39.177627Z","iopub.execute_input":"2022-07-27T00:31:39.178511Z","iopub.status.idle":"2022-07-27T00:31:39.203245Z","shell.execute_reply.started":"2022-07-27T00:31:39.178472Z","shell.execute_reply":"2022-07-27T00:31:39.202307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### MAIN FE","metadata":{"id":"EzVtgow-3PKw"}},{"cell_type":"code","source":"def get_not_used():  \n  return ['row_id', 'customer_ID', 'target', 'cid', 'S_2',\"D_103\",\"D_139\"]\n\nstats = ['mean', 'min', 'max','std']\nfeatures_avg = ['S_2_wk','B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_8', 'B_9', 'B_10', 'B_11', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18',\n                'B_19', 'B_20', 'B_21', 'B_22', 'B_23', 'B_24', 'B_25', 'B_28', 'B_29', 'B_30', 'B_32', 'B_33', 'B_37', 'B_38', 'B_39', 'B_40', 'B_41', 'B_42',\n                'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_50', 'D_51', 'D_53', 'D_54', 'D_55', 'D_58', 'D_59', 'D_60', 'D_61', \n                'D_62', 'D_65', 'D_66', 'D_69', 'D_70', 'D_71', 'D_72', 'D_73', 'D_74', 'D_75', 'D_76', 'D_77', 'D_78', 'D_80', 'D_82', 'D_84', 'D_86', 'D_91', \n                'D_92', 'D_94', 'D_96', 'D_103', 'D_104', 'D_108', 'D_112', 'D_113', 'D_114', 'D_115', 'D_117', 'D_118', 'D_119', 'D_120', 'D_121', 'D_122', 'D_123',\n                'D_124', 'D_125', 'D_126', 'D_128', 'D_129', 'D_131', 'D_132', 'D_133', 'D_134', 'D_135', 'D_136', 'D_140', 'D_141', 'D_142', 'D_144', 'D_145',\n                'P_2', 'P_3', 'P_4', 'R_1', 'R_2', 'R_3', 'R_7', 'R_8', 'R_9', 'R_10', 'R_11', 'R_14', 'R_15', 'R_16', 'R_17', 'R_20', 'R_21', 'R_22', 'R_24', \n                'R_26', 'R_27', 'S_3', 'S_5', 'S_6', 'S_7', 'S_9', 'S_11', 'S_12', 'S_13', 'S_15', 'S_16', 'S_18', 'S_22', 'S_23', 'S_25', 'S_26']\n#features_std = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_8', 'B_9', 'B_10', 'B_11', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_20', 'B_21', 'B_22', 'B_23', 'B_24', 'B_25', 'B_28', 'B_29', 'B_30', 'B_32', 'B_33', 'B_37', 'B_38', 'B_39', 'B_40', 'B_41', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_50', 'D_51', 'D_53', 'D_54', 'D_55', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_65', 'D_66', 'D_69', 'D_70', 'D_71', 'D_72', 'D_73', 'D_74', 'D_75', 'D_76', 'D_77', 'D_78', 'D_80', 'D_82', 'D_84', 'D_86', 'D_91', 'D_92', 'D_94', 'D_96', 'D_103', 'D_104', 'D_108', 'D_112', 'D_113', 'D_114', 'D_115', 'D_117', 'D_118', 'D_119', 'D_120', 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_126', 'D_128', 'D_129', 'D_131', 'D_132', 'D_133', 'D_134', 'D_135', 'D_136', 'D_140', 'D_141', 'D_142', 'D_144', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_2', 'R_3', 'R_7', 'R_8', 'R_9', 'R_10', 'R_11', 'R_14', 'R_15', 'R_16', 'R_17', 'R_20', 'R_21', 'R_22', 'R_24', 'R_26', 'R_27', 'S_3', 'S_5', 'S_6', 'S_7', 'S_9', 'S_11', 'S_12', 'S_13', 'S_15', 'S_16', 'S_18', 'S_22', 'S_23', 'S_25', 'S_26']\nfeatures_min = ['B_2', 'B_4', 'B_5', 'B_9', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_19', 'B_20', 'B_28', 'B_29', 'B_33', 'B_36', 'B_42', 'D_39',\n                'D_41', 'D_42', 'D_45', 'D_46', 'D_48', 'D_50', 'D_51', 'D_53', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_62', 'D_70', 'D_71', 'D_74', \n                'D_75', 'D_78', 'D_83', 'D_102', 'D_112', 'D_113', 'D_115', 'D_118', 'D_119', 'D_121', 'D_122', 'D_128', 'D_132', 'D_140', 'D_141', 'D_144',\n                'D_145', 'P_2', 'P_3', 'R_1', 'R_27', 'S_3', 'S_5', 'S_7', 'S_9', 'S_11', 'S_12', 'S_23', 'S_25']\nfeatures_max = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9', 'B_10', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19',\n                'B_21', 'B_23', 'B_24', 'B_25', 'B_29', 'B_30', 'B_33', 'B_37', 'B_38', 'B_39', 'B_40', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44',\n                'D_45', 'D_46', 'D_47', 'D_48', 'D_49', 'D_50', 'D_52', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_63', 'D_64', 'D_65', 'D_70',\n                'D_71', 'D_72', 'D_73', 'D_74', 'D_76', 'D_77', 'D_78', 'D_80', 'D_82', 'D_84', 'D_91', 'D_102', 'D_105', 'D_107', 'D_110', 'D_111', 'D_112',\n                'D_115', 'D_116', 'D_117', 'D_118', 'D_119', 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_126', 'D_128', 'D_131', 'D_132', 'D_133',\n                'D_134', 'D_135', 'D_136', 'D_138', 'D_140', 'D_141', 'D_142', 'D_144', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_3', 'R_5', 'R_6', 'R_7',\n                'R_8', 'R_10', 'R_11', 'R_14', 'R_17', 'R_20', 'R_26', 'R_27', 'S_3', 'S_5', 'S_7', 'S_8', 'S_11', 'S_12', 'S_13', 'S_15', 'S_16', 'S_22',\n                'S_23', 'S_24', 'S_25', 'S_26', 'S_27']\nfeatures_last = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9', 'B_10', 'B_11', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', \n                 'B_18', 'B_19', 'B_20', 'B_21', 'B_22', 'B_23', 'B_24', 'B_25', 'B_26', 'B_28', 'B_29', 'B_30', 'B_32', 'B_33', 'B_36', 'B_37', 'B_38',\n                 'B_39', 'B_40', 'B_41', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_49', 'D_50', 'D_51', 'D_52',\n                 'D_53', 'D_54', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_63', 'D_64', 'D_65', 'D_69', 'D_70', 'D_71', 'D_72', 'D_73', 'D_75', 'D_76', 'D_77', 'D_78', 'D_79', 'D_80', 'D_81', 'D_82', 'D_83', 'D_86', 'D_91', 'D_96', 'D_105', 'D_106', 'D_112', 'D_114', 'D_119', 'D_120', 'D_121', 'D_122', 'D_124', 'D_125', 'D_126', 'D_127', 'D_130', 'D_131', 'D_132', 'D_133', 'D_134', 'D_138', 'D_140', 'D_141', 'D_142', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_2', 'R_3', 'R_4', 'R_5', 'R_6', 'R_7', 'R_8', 'R_9', 'R_10', 'R_11', 'R_12', 'R_13', 'R_14', 'R_15', 'R_19', 'R_20', 'R_26', 'R_27', 'S_3',\n                 'S_5', 'S_6', 'S_7', 'S_8', 'S_9', 'S_11', 'S_12', 'S_13', 'S_16', 'S_19', 'S_20', 'S_22', 'S_23', 'S_24', 'S_25', 'S_26', 'S_27']\n# Feature Engineering on credit risk\nspend_p=[ 'S_3',  'S_5', 'S_6', 'S_7', 'S_8', 'S_9', 'S_11', 'S_12', 'S_13', 'S_15', 'S_16', 'S_17', 'S_18', 'S_19', 'S_20', 'S_22', 'S_23', 'S_24', 'S_25', 'S_26', 'S_27']\nbalance_p = ['B_1', 'B_2', 'B_3',  'B_5', 'B_6', 'B_7', 'B_8', 'B_9', 'B_10', 'B_11', 'B_12', 'B_13', 'B_14', 'B_15',  'B_17', 'B_18',  'B_21',   'B_23', 'B_24', 'B_25', 'B_26', 'B_27', 'B_28',  'B_36', 'B_37',  'B_40',    ]\npayment_p = ['P_2', 'P_3', 'P_4']\ncat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120',\n            'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\ndelq = ['D_39',\n                'D_41', 'D_42', 'D_45', 'D_46', 'D_48', 'D_50', 'D_51', 'D_53', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_62', 'D_70', 'D_71', 'D_74', \n                'D_75', 'D_78', 'D_83', 'D_102', 'D_112', 'D_113', 'D_115', 'D_118', 'D_119', 'D_121', 'D_122', 'D_128', 'D_132', 'D_140', 'D_141', 'D_144',\n                'D_145']                \ncat_cols_avg = [col for col in cat_cols if col in features_avg]\nnot_used = get_not_used()            \ng_num_cols = []\ndef preprocess(df):\n    df['row_id'] = cupy.arange(df.shape[0])\n    not_used = get_not_used()\n    # Drop cols https://www.kaggle.com/code/raddar/redundant-features-amex/notebook\n    #df=df.drop([\"D_103\",\"D_139\"],axis=1)\n    num_cols = [col for col in df.columns if col not in cat_cols+not_used]  \n\n    # Null columns handling  \n    # ref https://www.kaggle.com/code/illidan7/amex-basic-feature-engineering-1500-features/notebook\n    #nullvals = df.isnull().sum() / df.shape[0]\n    #nullCols = nullvals[nullvals>0.3].index.to_list()\n    #for col in nullCols:\n    #    df[col+'_null'] = df[col].isnull().astype(int) \n        \n    #exclnullCols = nullvals[nullvals>0.9].index.to_list()\n    #for col in exclnullCols:\n    #    df[col+'_exclnull'] = df[col].isnull().astype(int) \n    \n    #nullAggCols = [col + \"_null\" for col in nullCols]  \n    #exclnullCols = [col + '_exclnull' for col in exclnullCols]\n\n    globals()['g_num_cols'] = num_cols\n    for col in df.columns:\n        if col not in not_used+cat_cols:\n           df[col] = df[col].astype('float32').round(decimals=2).astype('float16') \n    print(f\"Starting fe [{len(df.columns)}]\") \n    dgs=add_stats_step(df, num_cols)\n    #custom stats  \n    if CFG.KURT:       \n      ## Replace with num_cols\n      train_stat = df.groupby(\"customer_ID\")[balance_p+spend_p+payment_p+delq].agg(kurtosis)\n      train_stat.columns = [x+'_kurt' for x in train_stat.columns]\n      print(train_stat.columns)\n      train_stat.reset_index(inplace = True)    \n      dgs.append(train_stat) \n      print(f\"Stats Kurt calc [{len(df.columns)}]\")    \n \n\n    train_stat = df.groupby(\"customer_ID\")[spend_p+payment_p+delq].agg('sum')\n    train_stat.columns = [x+'_sum' for x in train_stat.columns]\n    print(train_stat.columns)\n    train_stat.reset_index(inplace = True)    \n    dgs.append(train_stat)\n    del train_stat; gc.collect() \n    print(f\"Stats Sum calc [{len(df.columns)}]\")    \n    # END Custom cherry picked Stats\n    print(f\"Stats added and calculated [{len(df.columns)}]\")    \n\n    # Add s2 count as a feature ( Number of spends)\n    s2_count = df.groupby(\"customer_ID\")['S_2'].agg(['count']) \n    s2_count.columns = ['S_2_Count']\n    s2_count.reset_index(inplace = True)     \n    dgs.append(s2_count)\n    print(f\"Stats added and calculated [{len(s2_count.columns)}]\")    \n    del s2_count; gc.collect() \n\n    df[\"P_SUM\"] = df[payment_p].sum(axis=1) \n    df[\"S_SUM\"] = df[spend_p].sum(axis=1) \n    df[\"P-S\"] = df.P_SUM - df.S_SUM       \n    df=df.drop([\"S_SUM\",\"P_SUM\"],axis=1)\n    print(f\"P-S feature added\")      \n\n    # Add Lag Columns \n    if CFG.ADD_LAG:\n      train_num_agg = df.groupby(\"customer_ID\")[num_cols].agg(['first', 'last'])#payment_p+balance_p+spend_p\n      train_num_agg.columns = ['_'.join(x) for x in train_num_agg.columns]\n      train_num_agg.reset_index(inplace = True) \n      for col in train_num_agg:\n        if 'last' in col and col.replace('last', 'first') in train_num_agg:\n                    train_num_agg[col + '_lag_sub'] = train_num_agg[col] - train_num_agg[col.replace('last', 'first')]\n                    train_num_agg[col + '_lag_div'] = train_num_agg[col] / train_num_agg[col.replace('last', 'first')]            \n      train_num_agg.drop([col for col in train_num_agg.columns if \"last\" in col],axis=1, inplace=True)\n      dgs.append(train_num_agg)\n      del train_num_agg\n      # get_difference():\n      print(f\"Computing diff 1 features ,curr cols [{len(df.columns)}]\") \n      dff_cols =  payment_p+balance_p+spend_p+delq ## Replace with num_cols\n      if CFG.ADD_DIFF_1:\n        train_diff = df.loc[:,num_cols+['customer_ID']].groupby(['customer_ID']).apply(lambda x: cupy.diff(x.values[-2:,:], axis = 0).squeeze().astype(cupy.float32))\n        index = train_diff.index\n        cols = [col + '_diff1' for col in df[num_cols].columns]\n        train_diff = pd.DataFrame(train_diff.values.tolist(), columns=cols)   \n        train_diff['customer_ID'] = index    \n        train_diff.reset_index(inplace = True) \n        print(f\"Computing diff 1 features ,curr cols [{ train_diff.columns}]\")\n        dgs.append(train_diff) \n      for pdf in CFG.ADD_DIFF:\n        train_diff = get_difference(df, dff_cols,period=pdf)\n        print(f\"Computing Diff {pdf} ,curr cols [{ train_diff.columns}]\") \n        dgs.append(train_diff)    \n        del train_diff; gc.collect()        \n      for pdf in CFG.ADD_PCTDIFF:\n        train_diff = get_pct_change(df, dff_cols,period=pdf)\n        print(f\"Computing pct change {pdf} ,curr cols [{ train_diff.columns}]\") \n        dgs.append(train_diff)    \n        del train_diff; gc.collect() \n      print(f\"Lag Features added [{len(df.columns)}]\")          \n    \n    # compute \"after pay\" features\n    for bcol in [f'B_{i}' for i in [11,14,17]]+['D_39','D_131']+[f'S_{i}' for i in [16,23]]:\n        for pcol in ['P_2','P_3']:\n            if bcol in df.columns:\n                df[f'{bcol}-{pcol}'] = df[bcol] - df[pcol]\n    #\n    df['S_2'] = cudf.to_datetime(df['S_2'])\n    df['cid'], _ = df.customer_ID.factorize()    \n\n    # Add sundays count as a feature \n    s2_count = df[df.S_2.dt.dayofweek == 6].groupby(\"customer_ID\")['S_2'].agg(['count']) \n    s2_count.columns = ['S_2_Sun_Count']\n    s2_count.reset_index(inplace = True)     \n    dgs.append(s2_count)\n    print(f\"sundays count added and calculated [{len(s2_count.columns)}]\")    \n    # Add week of the month correlation\n    df['S_2_wk'] =  df['S_2'].dt.week\n    s2_count = df.groupby(\"customer_ID\")['S_2_wk'].agg(['std'])  \n    s2_count.columns = ['S_2_wk_std']\n    s2_count.reset_index(inplace = True)     \n    dgs.append(s2_count)\n    df=df.drop([\"S_2_wk\"],axis=1 )\n    print(f\"sundays count added and calculated [{len(s2_count.columns)}]\")        \n    del s2_count; gc.collect()     \n\n    ## Flatten Categoricals\n    for CAT_COL in cat_cols:\n      X_train_cat_pd = df[[CAT_COL,TIME,CID]] \n      te_fold = df.groupby([CID,TIME]).cumcount().to_dict() \n      X_train_pivot = pivot_data(X_train_cat_pd[[CID, TIME, CAT_COL]])\n      display(X_train_pivot.head(1))\n      dgs.append(X_train_pivot)\n      print(f\"Adding flattened cat_col {CAT_COL}\")\n      del X_train_pivot; gc.collect()   \n    del X_train_cat_pd\n\n\n    if CFG.ADD_CAT:\n      train_cat_agg = df.groupby(\"customer_ID\")[cat_cols].agg(['count', 'nunique', 'std']) \n      train_cat_agg.columns = ['_'.join(x) for x in train_cat_agg.columns]\n      train_cat_agg.reset_index(inplace = True)     \n      dgs.append(train_cat_agg)\n      del train_cat_agg; gc.collect() \n      train_cat_mean = df.groupby(\"customer_ID\")[cat_cols_avg].agg(['mean']) \n      train_cat_mean.columns = ['_'.join(x) for x in train_cat_mean.columns]\n      train_cat_mean.reset_index(inplace = True)    \n      print(f\"Added cat mean cols [{train_cat_mean.columns}]\")   \n      dgs.append(train_cat_mean)\n      del train_cat_mean; gc.collect() \n      print(f\"CAT features added {len(df.columns)}\") \n      \n    # cudf merge changes row orders\n    # restore the original row order by sorting row_id\n    df = df.sort_values('row_id')\n    df = df.drop(['row_id'],axis=1)\n    return df, dgs\n\ndef add_stats_step(df, cols):\n    n = 50\n    dgs = []\n    for i in range(0,len(cols),n):\n        s = i\n        e = min(s+n, len(cols))\n        dg = add_stats_one_shot(df, cols[s:e])\n        dgs.append(dg)\n    return dgs\n\ndef add_stats_one_shot(df, cols):\n    \n    dg = df.groupby('customer_ID').agg({col:stats for col in cols})\n    out_cols = []\n    for col in cols:\n        out_cols.extend([f'{col}_{s}' for s in stats])\n    dg.columns = out_cols\n    dg = dg.reset_index()\n    return dg\n\ndef load_test_iter(path, chunks=4):\n    \n    test_rows = 11363762\n    chunk_rows = test_rows // chunks\n    \n    test = cudf.read_parquet(f'{path}/test.parquet',\n                             columns=['customer_ID','S_2'],\n                             num_rows=test_rows)\n    test = get_segment(test)\n    start = 0\n    while start < test.shape[0]:\n        if start+chunk_rows < test.shape[0]:\n            end = test['cus_count'].values[start+chunk_rows]\n        else:\n            end = test['cus_count'].values[-1]\n        end = int(end)\n        df = cudf.read_parquet(f'{path}/test.parquet',\n                               num_rows = end-start, skiprows=start)\n        start = end\n        yield process_data(df)\n    \n\ndef load_train(path):\n    train = cudf.read_parquet(f'{path}/train.parquet')\n    \n    train = process_data(train)\n    trainl = cudf.read_csv(f'{CFG.INPUT}/amex-default-prediction/train_labels.csv')\n    train = train.merge(trainl, on='customer_ID', how='left')\n    return train\n\ndef process_data(df):\n    df,dgs = preprocess(df) \n    df = df.drop_duplicates('customer_ID',keep='last')\n    for dg in dgs:\n        df = df.merge(dg, on='customer_ID', how='left')\n        # drop specific non impactful cols \n    del dgs; gc.collect()    \n    if CFG.TRIM:\n      drop_col = [col for  col in df.columns if ((\"std\" in col) and (col.replace(\"_std\",\"\") not in features_avg))]\n      print(f\"Dropping {drop_col}\")\n      df=df.drop(drop_col,axis=1)      \n      drop_col = [col for  col in df.columns if ((\"min\" in col) and (col.replace(\"_min\",\"\") not in features_min))]\n      print(f\"Dropping {drop_col}\")\n      df=df.drop(drop_col,axis=1)\n      drop_col = [col for  col in df.columns if  ((\"max\" in col) and (col.replace(\"_max\",\"\") not in features_max))]\n      print(f\"Dropping {drop_col}\")\n      df=df.drop(drop_col,axis=1)\n      #drop_col = [col for  col in df.columns if  ((\"mean\" in col) and (col.replace(\"_mean\",\"\") not in features_avg))]\n      #print(f\"Dropping {drop_col}\")\n      #df=df.drop(drop_col,axis=1)\n      #drop_col = [col for  col in df.columns if  ((\"sum\" in col) and (col.replace(\"_sum\",\"\") not in features_min))]\n      #print(f\"Dropping {drop_col}\")       \n      #df=df.drop(drop_col,axis=1)                 \n    diff_cols = [col for col in df.columns if col.endswith('_diff')]\n    df = df.drop(diff_cols,axis=1)\n    print(f\"All stats merged {len(df.columns)}\")    \n    cat_features= [col for col in x.columns if \"__TE\" in col ]\n    df[cat_features] = df[cat_features].values.astype(int) \n    df[cat_features]= x[cat_features].fillna(-1) \n    # add mean - last features and More custom features\n    df[\"P2B9\"] = df[\"P_2\"] / df[\"B_9\"] \n    math_col = globals()['g_num_cols']\n    for pcol in math_col:\n      if pcol+\"_mean\" in df.columns:\n        df[f'{pcol}-mean'] = df[pcol] - df[pcol+\"_mean\"]\n        df[f'{pcol}-div-mean'] = df[pcol] /df[pcol+\"_mean\"]\n      if (pcol+\"_min\" in df.columns) and (pcol+\"_max\" in df.columns):  \n        df[f'{pcol}_min_div_max'] = df[pcol+\"_min\"] / df[pcol+\"_max\"]\n        df[f'{pcol}_min-max'] = df[pcol+\"_min\"] - df[pcol+\"_max\"]\n      # compute z score \n      if CFG.COMPUTE_Z:\n        if (pcol+\"_mean\" in df.columns) and (pcol+\"_std\" in df.columns):\n          df[f'{pcol}-zscore'] = (df[pcol] - df[pcol+\"_mean\"])/df[pcol+\"_std\"]\n          # Remove last as standardized\n          #df = df.drop(pcol,axis=1)\n    print(f\"Addding col-mean {len(df.columns)} cols {math_col}\")      \n    # Last payment/Min Payment (uncomment this)\n    for pcol in payment_p:\n      if pcol+\"_min\" in df.columns:  \n        df[f\"{pcol}_div_min\"] = df[f'{pcol}']/df[f'{pcol}_min']\n    # computer (spend-pay/balance) ratios\n    for scol in [f'S_{i}' for i in [16,23]]:\n        for pcol in ['P_2','P_3']:\n            for bcol in [f'B_{i}' for i in [11,14,17]]:\n                print(f\"Addding (spend-pay/balance) ratios {scol}-{pcol}div{bcol}\")     \n                df[f'{scol}-{pcol}div{bcol}'] = df[f'{scol}-{pcol}']/df[f'{bcol}']  \n\n    if CFG.ADD_DIFF:\n      for scol in [f'S_{i}' for i in [16,23]]:\n          for pcol in ['P_2','P_3']:\n              for bcol in [f'B_{i}' for i in [11,14,17]]:\n                  print(f\"Addding (spend-pay/balance)_sum ratios {scol}-{pcol}div{bcol}\")     \n                  df[f'{scol}s-{pcol}s_d_{bcol}'] = (df[f'{scol}_sum']-df[f'{pcol}_sum'])/df[f'{bcol}']                         \n    print(f\"Addding col-mean + custom features {len(features_avg)} cols {globals()['g_num_cols']}\") \n    main_features = [f'B_{i}' for i in [11,14,17]]+['D_39','D_131']+[f'S_{i}' for i in [16,23]]+['P_2','P_3']\n    cat_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68'] + ['B_31', 'D_87']\n    df = agg_global_rank(df,main_features)\n    print(f\"Added agg_global_rank\") \n    df = agg_pct_rank_by_cat(df,main_features,cat_features)   \n    print(f\"Added agg_pct_rank_by_cat\")         \n    #df = agg_standardize_by_cat(df,main_features,cat_features) \n    print(f\"Added agg_standardize_by_cat\") \n    df = df.drop(cat_cols,axis=1)      \n    return df\n\ndef get_segment(test):\n    dg = test.groupby('customer_ID').agg({'S_2':'count'})\n    dg.columns = ['cus_count']\n    dg = dg.reset_index()\n    dg['cid'],_ = dg['customer_ID'].factorize()\n    dg = dg.sort_values('cid')\n    dg['cus_count'] = dg['cus_count'].cumsum()\n    \n    test = test.merge(dg, on='customer_ID', how='left')\n    test = test.sort_values(['cid','S_2'])\n    assert test['cus_count'].values[-1] == test.shape[0]\n    return test","metadata":{"id":"IgC1UYiKgrx4","execution":{"iopub.status.busy":"2022-07-27T00:31:39.205160Z","iopub.execute_input":"2022-07-27T00:31:39.205545Z","iopub.status.idle":"2022-07-27T00:31:39.355772Z","shell.execute_reply.started":"2022-07-27T00:31:39.205509Z","shell.execute_reply":"2022-07-27T00:31:39.354875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CATBOOST Params and utility functions","metadata":{"id":"ksVFY-18grx6"}},{"cell_type":"code","source":"class AmexMetric(object):\n    def get_final_error(self, error, weight):\n        return error / (weight + 1e-38)\n\n    def is_max_optimal(self):\n        return True\n\n    def evaluate(self, approxes, target, weight):\n        # approxes - list of list-like objects (one object per      approx dimension)\n        # target - list-like object\n        # weight - list-like object, can be None\n        assert len(approxes) == 1\n        assert len(target) == len(approxes[0])\n        \n        approx = approxes[0]\n        pred = [0 for i in range(len(target))] \n        return amex_metric(np.array(target), pred), 0","metadata":{"id":"yD1ZqJNszRTA","execution":{"iopub.status.busy":"2022-07-27T00:31:39.365923Z","iopub.execute_input":"2022-07-27T00:31:39.368258Z","iopub.status.idle":"2022-07-27T00:31:39.382014Z","shell.execute_reply.started":"2022-07-27T00:31:39.368221Z","shell.execute_reply":"2022-07-27T00:31:39.381087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def cat_train(x, y, xt, yt,\n               cat_features=['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120',\n                'D_126', 'D_63', 'D_64', 'D_66', 'D_68'],#+[cat+\"_first\" for cat in cat_cols],\n                params = {\"iterations\":10000}): \n    #print(params)            \n    #params = {'iterations': 4885, 'l2_leaf_reg': 3, 'bootstrap_type': 'Bernoulli', 'max_depth': 10, 'subsample': 0.16284093343361972}\n    cat_features= [col for col in x.columns if \"__TE\" in col ] \n    print(f\"cat_features {cat_features}\")\n    model = CatBoostClassifier( random_state=CFG.seed, #nan_mode='Min',\n                                task_type=\"GPU\",\n                                devices='0:1', \n                                iterations = 10500,\n                                #learning_rate = 0.01,\n                                #used_ram_limit=2*1024*1024*1024,\n                                #pinned_memory_size=2*1024*1024*1024,\n                                depth = 9,\n                                #eval_metric= AmexMetric,\n                                **params)\n    model.fit(x, y, eval_set=[(xt, yt)], cat_features=cat_features,\n              verbose=100, early_stopping_rounds=700)\n    return model.predict_proba(xt)[:, 1],model,1","metadata":{"id":"coOPXOi_grx7","execution":{"iopub.status.busy":"2022-07-27T00:31:39.386355Z","iopub.execute_input":"2022-07-27T00:31:39.390565Z","iopub.status.idle":"2022-07-27T00:31:39.401768Z","shell.execute_reply.started":"2022-07-27T00:31:39.390532Z","shell.execute_reply":"2022-07-27T00:31:39.400773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Metrics","metadata":{"id":"eUS5KsVDgrx8"}},{"cell_type":"code","source":"def xgb_amex(y_pred, y_true):\n    return 'amex', amex_metric_np(y_pred,y_true.get_label())\n\n# Created by https://www.kaggle.com/yunchonggan\n# https://www.kaggle.com/competitions/amex-default-prediction/discussion/328020\ndef amex_metric_np(preds: np.ndarray, target: np.ndarray) -> float:\n    indices = np.argsort(preds)[::-1]\n    preds, target = preds[indices], target[indices]\n\n    weight = 20.0 - target * 19.0\n    cum_norm_weight = (weight / weight.sum()).cumsum()\n    four_pct_mask = cum_norm_weight <= 0.04\n    d = np.sum(target[four_pct_mask]) / np.sum(target)\n\n    weighted_target = target * weight\n    lorentz = (weighted_target / weighted_target.sum()).cumsum()\n    gini = ((lorentz - cum_norm_weight) * weight).sum()\n\n    n_pos = np.sum(target)\n    n_neg = target.shape[0] - n_pos\n    gini_max = 10 * n_neg * (n_pos + 20 * n_neg - 19) / (n_pos + 20 * n_neg)\n\n    g = gini / gini_max\n    return 0.5 * (g + d)\n\n# we still need the official metric since the faster version above is slightly off\nimport pandas as pd\ndef amex_metric(y_true, y_pred):\n    labels = np.transpose(np.array([y_true, y_pred]))\n    labels = labels[labels[:, 1].argsort()[::-1]]\n    weights = np.where(labels[:,0]==0, 20, 1)\n    cut_vals = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n    gini = [0,0]\n    for i in [1,0]:\n        labels = np.transpose(np.array([y_true, y_pred]))\n        labels = labels[labels[:, i].argsort()[::-1]]\n        weight = np.where(labels[:,0]==0, 20, 1)\n        weight_random = np.cumsum(weight / np.sum(weight))\n        total_pos = np.sum(labels[:, 0] *  weight)\n        cum_pos_found = np.cumsum(labels[:, 0] * weight)\n        lorentz = cum_pos_found / total_pos\n        gini[i] = np.sum((lorentz - weight_random) * weight)\n    return 0.5 * (gini[1]/gini[0] + top_four)\n\ndef lgb_amex_metric(y_pred, y_true):\n    y_true = y_true.get_label()\n    return 'amex_metric', amex_metric(y_true, y_pred), True","metadata":{"id":"KiUMoni5grx8","execution":{"iopub.status.busy":"2022-07-27T00:31:39.402991Z","iopub.execute_input":"2022-07-27T00:31:39.403560Z","iopub.status.idle":"2022-07-27T00:31:39.428036Z","shell.execute_reply.started":"2022-07-27T00:31:39.403488Z","shell.execute_reply":"2022-07-27T00:31:39.427198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load data and add feature","metadata":{"id":"e8ZNozN7grx9"}},{"cell_type":"code","source":"%%time \nimport time \nimport torch, gc \ntorch.cuda.empty_cache()\ngc.collect()    \n\nif CFG.TRAIN or CFG.OPTIMIZE:\n  fe = \"/content/train_fe_v1658764246.287694.pickle\"\n  if os.path.exists(fe):\n    train = pd.read_pickle(fe) \n  else:  \n    path = f'{CFG.INPUT}/amex-data-integer-dtypes-parquet-format'\n    train = load_train(path)    \n    features = [col for col in train.columns if col not in  get_not_used()] \n    print(\"Saving FE to file\")    \n    train.to_pickle(f\"train_fe_v{time.time()}.pickle\")\n  print(train.shape)\n  train.head()","metadata":{"id":"W3fHX37Igrx-","outputId":"0f394807-25b8-4d17-d0e0-f153fdd08edb","execution":{"iopub.status.busy":"2022-07-27T00:31:39.431778Z","iopub.execute_input":"2022-07-27T00:31:39.434420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.DEBUG:\n  train = train.sample(n=2000, random_state=42).reset_index(drop=True)\n  features = [col for col in train.columns if col not in  get_not_used()]\nif CFG.INFER:\n  test = process_data( cudf.read_parquet(f'/content/amex-data-integer-dtypes-parquet-format/test.parquet' ))\n  print(f\"Test size {len(test)}\")\n  features = [col for col in test.columns if col not in  get_not_used()]  \n_ = gc.collect()","metadata":{"id":"7YVypOzegrx-","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train CAT_BOOST","metadata":{"id":"0y1TLKFDgrx_"}},{"cell_type":"code","source":"not_used = get_not_used()\nmsgs = {}\nfolds = CFG.n_folds\nscore = 0\n\nkfold = StratifiedKFold(n_splits = CFG.n_folds, shuffle = True, random_state = CFG.seed) \n\ndef train_fn(fold,x,y,xt,yt,_params= {}):   \n    print(\"Start training\")\n    val_pred,model, bst = cat_train(x, y, xt, yt,params=_params)  \n    return val_pred,model\n\nif CFG.TRAIN:  \n  features = [col for col in train.columns if col not in  get_not_used()]\n  oof_predictions = np.zeros(len(train))\n  feature_importances = pd.DataFrame()\n  feature_importances[\"feature\"] = features\n  for fold, (trn_ind, val_ind) in enumerate(kfold.split(train, train[CFG.target])):   \n      x, y = train[features].iloc[trn_ind], train[CFG.target].iloc[trn_ind]\n      xt, yt= train[features].iloc[val_ind], train[CFG.target].iloc[val_ind]\n      cat_features= [col for col in x.columns if \"__TE\" in col ]\n      x[cat_features] = x[cat_features].values.astype(int)\n      xt[cat_features] = xt[cat_features].values.astype(int)\n      x[cat_features]= x[cat_features].fillna(-1)\n      xt[cat_features]= xt[cat_features].fillna(-1)\n      if os.path.exists(f\"{CFG.model_dir}/1cat_fold{fold}_seed{CFG.seed}.pkl\"):\n        model = joblib.load(f\"{CFG.model_dir}/1cat_fold{fold}_seed{CFG.seed}.pkl\")\n        val_pred = model.predict_proba(xt)[:, 1] \n      else: \n        val_pred,model=train_fn(fold,x,y,xt,yt)\n        joblib.dump(model, f'cat_fold{fold}_seed{CFG.seed}.pkl')\n        \n      amex_score = amex_metric(yt.values,val_pred) \n      msg = f\"Fold {fold} amex {amex_score:.4f}\"          \n      oof_predictions[val_ind] = val_pred\n      feature_importances[f\"importance_fold{fold}+1\"] = model.feature_importances_\n      print(msg)\n      score += amex_score  \n      del x,y,xt,yt; gc.collect()\n  oof_df = pd.DataFrame({'customer_ID': train['customer_ID'], 'target': train[CFG.target], 'prediction': oof_predictions})\n  display(oof_df.head())\n  oof_df.to_csv(f'cat_{CFG.n_folds}fold_seed{CFG.seed}.csv', index = False)    \n  feature_importances .to_csv(f'feature_importances_{CFG.n_folds}fold_seed{CFG.seed}.csv', index = False) \n  score /= folds\n  print(f\"Average amex score: {score:.4f}\") \n      \nif CFG.INFER:\n  test_predictions = np.zeros(len(test))\n  not_used = [i for i in not_used if i in test.columns]\n  for fold  in range(CFG.n_folds):\n    model = joblib.load(f'cat_fold{fold}_seed{CFG.seed}.pkl')\n    test_pred = model.predict_proba(test[features])[:, 1]\n    test_predictions += test_pred / CFG.n_folds   \n    torch.cuda.empty_cache() \n  test_df = pd.DataFrame({'customer_ID': test['customer_ID'], 'prediction': test_predictions})\n  test_df.to_csv(f'test_cat_{CFG.n_folds}fold_seed{CFG.seed}.csv', index = False) ","metadata":{"id":"OyhJebfRt-VT","outputId":"9fb8d792-64d5-4dc8-81d1-6943236b9e69","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns \nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\nplt.rcParams['figure.figsize'] = (16,9)\nplt.rcParams[\"figure.facecolor\"] = '#FFFACD'\nplt.rcParams[\"axes.facecolor\"] = '#FFFFE0'\nplt.rcParams[\"axes.grid\"] = True \nplt.rcParams[\"grid.alpha\"] = 0.5\nplt.rcParams[\"grid.linestyle\"] = '--'\n\nfeature_importances['mean_importance']=feature_importances[[f'importance_fold{fold_n}+1' for fold_n in range(CFG.n_folds)]].mean(axis=1)\nfeature_importances.sort_values(by='mean_importance', ascending=False, inplace=True)\nsns.barplot(y=feature_importances['feature'][:50],x=feature_importances['mean_importance'][:50], palette='inferno')\nplt.title('Mean Feature Importance by Folds')\nplt.show()","metadata":{"id":"crYyQvJ4wG-a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Optuna","metadata":{"id":"SgQ8Pd9aSjaj"}},{"cell_type":"code","source":"import optuna \nfrom optuna.integration import CatBoostPruningCallback\n\n\ndef optunaOpt(model_name,t_params,n_trials=100, callbacks=(lambda trial: [])):\n    \"\"\" Best model eval util using Optuna\n    \"\"\"\n    def run(trials):\n        \"\"\" Optima trials lambda\"\"\"  \n        trial_params = {param:param_fn(trials) for param,param_fn in t_params.items()}  \n        if trial_params[\"bootstrap_type\"] == \"Bayesian\":\n          trial_params[\"bagging_temperature\"] =trials.suggest_float(\"bagging_temperature\", 0, 10)\n        if trial_params[\"bootstrap_type\"] == \"Bernoulli\":  \n          trial_params[\"subsample\"] =trials.suggest_float(\"subsample\", 0.1, 1)\n        not_used = get_not_used()\n        not_used = [i for i in not_used if i in train.columns]\n        for fold, (trn_ind, val_ind) in enumerate(kfold.split(train, train[CFG.target])):   \n            x, y = train[features].iloc[trn_ind], train[CFG.target].iloc[trn_ind]\n            xt, yt= train[features].iloc[val_ind], train[CFG.target].iloc[val_ind] \n            val_pred,model=train_fn(fold,x,y,xt,yt,trial_params) \n            break\n        \n        amex_score = amex_metric(yt.values,val_pred)\n        return amex_score\n    \n    study = optuna.create_study(direction=\"maximize\",\n                                study_name=f\"{model_name}-study\")\n    study.optimize(run, n_trials)\n    print('\\n Best Trial:')\n    print(study.best_trial)\n    print('\\n Best value')\n    print(study.best_value)\n    print('\\n Best hyperparameters:')\n    print(study.best_params)\n    return study ","metadata":{"id":"__T9yeWaSia8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 0.7932\ncatb_params = {\n    #\"iterations\":lambda trial :trial.suggest_int(\"iterations\", 6000, 11000), \n    #\"learning_rate\":lambda trial :trial.suggest_loguniform(\"learning_rate\", 0.1,1.0), \n    'l2_leaf_reg' : lambda trial :trial.suggest_categorical('l2_leaf_reg',[0.2,0.5,1,3]),\n    \"boosting_type\": lambda trial :trial.suggest_categorical(\"boosting_type\", [\"Ordered\", \"Plain\"]),\n    \"bootstrap_type\": lambda trial:trial.suggest_categorical(\n            \"bootstrap_type\", [\"Bayesian\", \"Bernoulli\" ]\n        ),\n    #\"colsample_bylevel\": lambda trial:trial.suggest_float(\"colsample_bylevel\", 0.01, 0.1, log=True),\n    \"depth\":lambda trial :trial.suggest_int(\"max_depth\", 7, 12),   \n}\n\nif CFG.OPTIMIZE:\n  optunaOpt(\"Catboost\",catb_params,n_trials=100, callbacks=(lambda trial: []))","metadata":{"id":"U8JeQHrIPXjQ","trusted":true},"execution_count":null,"outputs":[]}]}