{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Objective\n\nThe only boost compared to public notebooks is to use dart boosting and optimal hyperparammeters. Code run in my colab, just change the corresponding paths and uncomment and it should work, I uploaded test predictions to avoid running training and inference.","metadata":{}},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nimport itertools\n\n# ====================================================\n# Read & preprocess data and save it to disk\n# ====================================================\ndef read_preprocess_data():\n    train = pd.read_parquet('/content/data/train.parquet')\n    features = train.drop(['customer_ID', 'S_2'], axis = 1).columns.to_list()\n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\",\n    ]\n    num_features = [col for col in features if col not in cat_features]\n    print('Starting training feature engineer...')\n    train_num_agg = train.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    train_num_agg.columns = ['_'.join(x) for x in train_num_agg.columns]\n    train_num_agg.reset_index(inplace = True)\n    train_cat_agg = train.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    train_cat_agg.columns = ['_'.join(x) for x in train_cat_agg.columns]\n    train_cat_agg.reset_index(inplace = True)\n    train_labels = pd.read_csv('/content/data/train_labels.csv')\n    train = train_num_agg.merge(train_cat_agg, how = 'inner', on = 'customer_ID').merge(train_labels, how = 'inner', on = 'customer_ID')\n    del train_num_agg, train_cat_agg\n    gc.collect()\n    test = pd.read_parquet('/content/data/test.parquet')\n    print('Starting test feature engineer...')\n    test_num_agg = test.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\n    test_num_agg.reset_index(inplace = True)\n    test_cat_agg = test.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    test_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\n    test_cat_agg.reset_index(inplace = True)\n    test = test_num_agg.merge(test_cat_agg, how = 'inner', on = 'customer_ID')\n    del test_num_agg, test_cat_agg\n    gc.collect()\n    # Save files to disk\n    train.to_parquet('/content/drive/MyDrive/Amex/train_fe.parquet')\n    test.to_parquet('/content/drive/MyDrive/Amex/test_fe.parquet')\n\n# Read & Preprocess Data\n# read_preprocess_data()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training & Inference","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport os\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\nimport random\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nimport joblib\nimport itertools\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\nfrom itertools import combinations\n\n# ====================================================\n# Configurations\n# ====================================================\nclass CFG:\n    input_dir = '/content/data/'\n    seed = 42\n    n_folds = 5\n    target = 'target'\n\n# ====================================================\n# Seed everything\n# ====================================================\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\n# ====================================================\n# Read data\n# ====================================================\ndef read_data():\n    train = pd.read_parquet(CFG.input_dir + 'train_fe.parquet')\n    test = pd.read_parquet(CFG.input_dir + 'test_fe.parquet')\n    return train, test\n\n# ====================================================\n# Amex metric\n# ====================================================\ndef amex_metric(y_true, y_pred):\n    labels = np.transpose(np.array([y_true, y_pred]))\n    labels = labels[labels[:, 1].argsort()[::-1]]\n    weights = np.where(labels[:,0]==0, 20, 1)\n    cut_vals = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n    gini = [0,0]\n    for i in [1,0]:\n        labels = np.transpose(np.array([y_true, y_pred]))\n        labels = labels[labels[:, i].argsort()[::-1]]\n        weight = np.where(labels[:,0]==0, 20, 1)\n        weight_random = np.cumsum(weight / np.sum(weight))\n        total_pos = np.sum(labels[:, 0] *  weight)\n        cum_pos_found = np.cumsum(labels[:, 0] * weight)\n        lorentz = cum_pos_found / total_pos\n        gini[i] = np.sum((lorentz - weight_random) * weight)\n    return 0.5 * (gini[1]/gini[0] + top_four)\n\n# ====================================================\n# LGBM amex metric\n# ====================================================\ndef lgb_amex_metric(y_pred, y_true):\n    y_true = y_true.get_label()\n    return 'amex_metric', amex_metric(y_true, y_pred), True\n\n# ====================================================\n# Train & Evaluate\n# ====================================================\ndef train_and_evaluate(train, test):\n    # Label encode categorical features\n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\"\n    ]\n    cat_features = [f\"{cf}_last\" for cf in cat_features]\n    for cat_col in cat_features:\n        encoder = LabelEncoder()\n        train[cat_col] = encoder.fit_transform(train[cat_col])\n        test[cat_col] = encoder.transform(test[cat_col])\n    # Round last float features to 2 decimal place\n    num_cols = list(train.dtypes[(train.dtypes == 'float32') | (train.dtypes == 'float64')].index)\n    num_cols = [col for col in num_cols if 'last' in col]\n    for col in num_cols:\n        train[col + '_round2'] = train[col].round(2)\n        test[col + '_round2'] = test[col].round(2)\n    # Get feature list\n    features = [col for col in train.columns if col not in ['customer_ID', CFG.target]]\n    params = {\n        'objective': 'binary',\n        'metric': \"binary_logloss\",\n        'boosting': 'dart',\n        'seed': CFG.seed,\n        'num_leaves': 100,\n        'learning_rate': 0.01,\n        'feature_fraction': 0.20,\n        'bagging_freq': 10,\n        'bagging_fraction': 0.50,\n        'n_jobs': -1,\n        'lambda_l2': 2,\n        'min_data_in_leaf': 40\n        }\n    # Create a numpy array to store test predictions\n    test_predictions = np.zeros(len(test))\n    # Create a numpy array to store out of folds predictions\n    oof_predictions = np.zeros(len(train))\n    kfold = StratifiedKFold(n_splits = CFG.n_folds, shuffle = True, random_state = CFG.seed)\n    for fold, (trn_ind, val_ind) in enumerate(kfold.split(train, train[CFG.target])):\n        print(' ')\n        print('-'*50)\n        print(f'Training fold {fold} with {len(features)} features...')\n        x_train, x_val = train[features].iloc[trn_ind], train[features].iloc[val_ind]\n        y_train, y_val = train[CFG.target].iloc[trn_ind], train[CFG.target].iloc[val_ind]\n        lgb_train = lgb.Dataset(x_train, y_train, categorical_feature = cat_features)\n        lgb_valid = lgb.Dataset(x_val, y_val, categorical_feature = cat_features)\n        model = lgb.train(\n            params = params,\n            train_set = lgb_train,\n            num_boost_round = 10500,\n            valid_sets = [lgb_train, lgb_valid],\n            early_stopping_rounds = 100,\n            verbose_eval = 500,\n            feval = lgb_amex_metric\n            )\n        # Save best model\n        joblib.dump(model, f'/content/drive/MyDrive/Amex/Models/lgbm_fold{fold}_seed{CFG.seed}.pkl')\n        # Predict validation\n        val_pred = model.predict(x_val)\n        # Add to out of folds array\n        oof_predictions[val_ind] = val_pred\n        # Predict the test set\n        test_pred = model.predict(test[features])\n        test_predictions += test_pred / CFG.n_folds\n        # Compute fold metric\n        score = amex_metric(y_val, val_pred)\n        print(f'Our fold {fold} CV score is {score}')\n        del x_train, x_val, y_train, y_val, lgb_train, lgb_valid\n        gc.collect()\n    # Compute out of folds metric\n    score = amex_metric(train[CFG.target], oof_predictions)\n    print(f'Our out of folds CV score is {score}')\n    # Create a dataframe to store out of folds predictions\n    oof_df = pd.DataFrame({'customer_ID': train['customer_ID'], 'target': train[CFG.target], 'prediction': oof_predictions})\n    oof_df.to_csv(f'/content/drive/MyDrive/Amex/OOF/oof_lgbm_baseline_{CFG.n_folds}fold_seed{CFG.seed}.csv', index = False)\n    # Create a dataframe to store test prediction\n    test_df = pd.DataFrame({'customer_ID': test['customer_ID'], 'prediction': test_predictions})\n    test_df.to_csv(f'/content/drive/MyDrive/Amex/Predictions/test_lgbm_baseline_{CFG.n_folds}fold_seed{CFG.seed}.csv', index = False)\n    \n# seed_everything(CFG.seed)\n# train, test = read_data()\n# train_and_evaluate(train, test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Submission File\n\nThis is the submission file corresponding to the output of the previous pipeline","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('../input/amex-sub/test_lgbm_baseline_5fold_seed42.csv')\nsub.to_csv('test_lgbm_baseline_5fold_seed42.csv', index = False)","metadata":{},"execution_count":null,"outputs":[]}]}