{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Comments:\n    \nThis is an improvement of my baseline, you can find it here: https://www.kaggle.com/code/ragnar123/amex-lgbm-dart-cv-0-7963\n\nThe main difference between this solution and previous one is that we add new features and do seed blend to boost LB. Single 5 kfold model using seed 42 achieve an out of folds CV of 0.7977 and a public leaderboard of 0.799. If we use seed blend (train three different models using seed 42, 52, 62 and then average predictions) the LB boost niceley.\n\nThe main features that boost CV are the following:\n\n* The difference between last value and the lag1\n* The difference between last value and the average (this features gives a nice boost)\n\nThis feature engineer is done on all the last columns, so we actually add a lot of features, this model used 1368 features.\n\nI uploaded test predictions to avoid running training and inference\n\nNext Steps:\n\n* Could try feature selection, maybe a lot of the feature are just noise, actually I perform permutation importance and I reduce the amount of features to 1000 app and the CV was almost the same. Maybe there is a better feature selection technique that can boost performance.\n\n* Could try different models, maybe some neural network with the same features or a subset of the features and then blend with LGBM can work, in my experience blending tree models and neural network works great because they are very diverse so the boost is nice\n\n* Could try more feature engineering, maybe we can create more features that extract the hidden signal of the dataset, actually I would first work on this option and really try to capture all the signal that the dataset has.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nimport itertools\n\n# ====================================================\n# Get the difference\n# ====================================================\ndef get_difference(data, num_features):\n    df1 = []\n    customer_ids = []\n    for customer_id, df in tqdm(data.groupby(['customer_ID'])):\n        # Get the differences\n        diff_df1 = df[num_features].diff(1).iloc[[-1]].values.astype(np.float32)\n        # Append to lists\n        df1.append(diff_df1)\n        customer_ids.append(customer_id)\n    # Concatenate\n    df1 = np.concatenate(df1, axis = 0)\n    # Transform to dataframe\n    df1 = pd.DataFrame(df1, columns = [col + '_diff1' for col in df[num_features].columns])\n    # Add customer id\n    df1['customer_ID'] = customer_ids\n    return df1\n\n# ====================================================\n# Read & preprocess data and save it to disk\n# ====================================================\ndef read_preprocess_data():\n    train = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/train.parquet')\n    features = train.drop(['customer_ID', 'S_2'], axis = 1).columns.to_list()\n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\",\n    ]\n    num_features = [col for col in features if col not in cat_features]\n    print('Starting training feature engineer...')\n    train_num_agg = train.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    train_num_agg.columns = ['_'.join(x) for x in train_num_agg.columns]\n    train_num_agg.reset_index(inplace = True)\n    train_cat_agg = train.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    train_cat_agg.columns = ['_'.join(x) for x in train_cat_agg.columns]\n    train_cat_agg.reset_index(inplace = True)\n    train_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')\n    # Transform float64 columns to float32\n    cols = list(train_num_agg.dtypes[train_num_agg.dtypes == 'float64'].index)\n    for col in tqdm(cols):\n        train_num_agg[col] = train_num_agg[col].astype(np.float32)\n    # Transform int64 columns to int32\n    cols = list(train_cat_agg.dtypes[train_cat_agg.dtypes == 'int64'].index)\n    for col in tqdm(cols):\n        train_cat_agg[col] = train_cat_agg[col].astype(np.int32)\n    # Get the difference\n    train_diff = get_difference(train, num_features)\n    train = train_num_agg.merge(train_cat_agg, how = 'inner', on = 'customer_ID').merge(train_diff, how = 'inner', on = 'customer_ID').merge(train_labels, how = 'inner', on = 'customer_ID')\n    del train_num_agg, train_cat_agg, train_diff\n    gc.collect()\n    test = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet')\n    print('Starting test feature engineer...')\n    test_num_agg = test.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\n    test_num_agg.reset_index(inplace = True)\n    test_cat_agg = test.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    test_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\n    test_cat_agg.reset_index(inplace = True)\n    # Transform float64 columns to float32\n    cols = list(test_num_agg.dtypes[test_num_agg.dtypes == 'float64'].index)\n    for col in tqdm(cols):\n        test_num_agg[col] = test_num_agg[col].astype(np.float32)\n    # Transform int64 columns to int32\n    cols = list(test_cat_agg.dtypes[test_cat_agg.dtypes == 'int64'].index)\n    for col in tqdm(cols):\n        test_cat_agg[col] = test_cat_agg[col].astype(np.int32)\n    # Get the difference\n    test_diff = get_difference(test, num_features)\n    test = test_num_agg.merge(test_cat_agg, how = 'inner', on = 'customer_ID').merge(test_diff, how = 'inner', on = 'customer_ID')\n    del test_num_agg, test_cat_agg, test_diff\n    gc.collect()\n    # Save files to disk\n    train.to_parquet('./train_fe.parquet')\n    test.to_parquet('./test_fe.parquet')\n\n# Read & Preprocess Data\nread_preprocess_data()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:29:07.172805Z","iopub.execute_input":"2022-07-14T14:29:07.173606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training & Inference","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport os\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\nimport random\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nimport joblib\nimport itertools\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\nfrom itertools import combinations\n\n# ====================================================\n# Configurations\n# ====================================================\nclass CFG:\n    input_dir = '/content/data/'\n    seed = 42\n    n_folds = 5\n    target = 'target'\n    boosting_type = 'dart'\n    metric = 'binary_logloss'\n\n# ====================================================\n# Seed everything\n# ====================================================\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\n# ====================================================\n# Read data\n# ====================================================\ndef read_data():\n    train = pd.read_parquet(CFG.input_dir + 'train_fe.parquet')\n    test = pd.read_parquet(CFG.input_dir + 'test_fe.parquet')\n    return train, test\n\n# ====================================================\n# Amex metric\n# ====================================================\ndef amex_metric(y_true, y_pred):\n    labels = np.transpose(np.array([y_true, y_pred]))\n    labels = labels[labels[:, 1].argsort()[::-1]]\n    weights = np.where(labels[:,0]==0, 20, 1)\n    cut_vals = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n    gini = [0,0]\n    for i in [1,0]:\n        labels = np.transpose(np.array([y_true, y_pred]))\n        labels = labels[labels[:, i].argsort()[::-1]]\n        weight = np.where(labels[:,0]==0, 20, 1)\n        weight_random = np.cumsum(weight / np.sum(weight))\n        total_pos = np.sum(labels[:, 0] *  weight)\n        cum_pos_found = np.cumsum(labels[:, 0] * weight)\n        lorentz = cum_pos_found / total_pos\n        gini[i] = np.sum((lorentz - weight_random) * weight)\n    return 0.5 * (gini[1]/gini[0] + top_four)\n\n# ====================================================\n# LGBM amex metric\n# ====================================================\ndef lgb_amex_metric(y_pred, y_true):\n    y_true = y_true.get_label()\n    return 'amex_metric', amex_metric(y_true, y_pred), True\n\n# ====================================================\n# Train & Evaluate\n# ====================================================\ndef train_and_evaluate(train, test):\n    # Label encode categorical features\n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\"\n    ]\n    cat_features = [f\"{cf}_last\" for cf in cat_features]\n    for cat_col in cat_features:\n        encoder = LabelEncoder()\n        train[cat_col] = encoder.fit_transform(train[cat_col])\n        test[cat_col] = encoder.transform(test[cat_col])\n    # Round last float features to 2 decimal place\n    num_cols = list(train.dtypes[(train.dtypes == 'float32') | (train.dtypes == 'float64')].index)\n    num_cols = [col for col in num_cols if 'last' in col]\n    for col in num_cols:\n        train[col + '_round2'] = train[col].round(2)\n        test[col + '_round2'] = test[col].round(2)\n    # Get the difference between last and mean\n    num_cols = [col for col in train.columns if 'last' in col]\n    num_cols = [col[:-5] for col in num_cols if 'round' not in col]\n    for col in num_cols:\n        try:\n            train[f'{col}_last_mean_diff'] = train[f'{col}_last'] - train[f'{col}_mean']\n            test[f'{col}_last_mean_diff'] = test[f'{col}_last'] - test[f'{col}_mean']\n        except:\n            pass\n    # Transform float64 and float32 to float16\n    num_cols = list(train.dtypes[(train.dtypes == 'float32') | (train.dtypes == 'float64')].index)\n    for col in tqdm(num_cols):\n        train[col] = train[col].astype(np.float16)\n        test[col] = test[col].astype(np.float16)\n    # Get feature list\n    features = [col for col in train.columns if col not in ['customer_ID', CFG.target]]\n    params = {\n        'objective': 'binary',\n        'metric': CFG.metric,\n        'boosting': CFG.boosting_type,\n        'seed': CFG.seed,\n        'num_leaves': 100,\n        'learning_rate': 0.01,\n        'feature_fraction': 0.20,\n        'bagging_freq': 10,\n        'bagging_fraction': 0.50,\n        'n_jobs': -1,\n        'lambda_l2': 2,\n        'min_data_in_leaf': 40,\n        }\n    # Create a numpy array to store test predictions\n    test_predictions = np.zeros(len(test))\n    # Create a numpy array to store out of folds predictions\n    oof_predictions = np.zeros(len(train))\n    kfold = StratifiedKFold(n_splits = CFG.n_folds, shuffle = True, random_state = CFG.seed)\n    for fold, (trn_ind, val_ind) in enumerate(kfold.split(train, train[CFG.target])):\n        print(' ')\n        print('-'*50)\n        print(f'Training fold {fold} with {len(features)} features...')\n        x_train, x_val = train[features].iloc[trn_ind], train[features].iloc[val_ind]\n        y_train, y_val = train[CFG.target].iloc[trn_ind], train[CFG.target].iloc[val_ind]\n        lgb_train = lgb.Dataset(x_train, y_train, categorical_feature = cat_features)\n        lgb_valid = lgb.Dataset(x_val, y_val, categorical_feature = cat_features)\n        model = lgb.train(\n            params = params,\n            train_set = lgb_train,\n            num_boost_round = 10500,\n            valid_sets = [lgb_train, lgb_valid],\n            early_stopping_rounds = 1500,\n            verbose_eval = 500,\n            feval = lgb_amex_metric\n            )\n        # Save best model\n        joblib.dump(model, f'/content/drive/MyDrive/Amex/Models/lgbm_{CFG.boosting_type}_fold{fold}_seed{CFG.seed}.pkl')\n        # Predict validation\n        val_pred = model.predict(x_val)\n        # Add to out of folds array\n        oof_predictions[val_ind] = val_pred\n        # Predict the test set\n        test_pred = model.predict(test[features])\n        test_predictions += test_pred / CFG.n_folds\n        # Compute fold metric\n        score = amex_metric(y_val, val_pred)\n        print(f'Our fold {fold} CV score is {score}')\n        del x_train, x_val, y_train, y_val, lgb_train, lgb_valid\n        gc.collect()\n    # Compute out of folds metric\n    score = amex_metric(train[CFG.target], oof_predictions)\n    print(f'Our out of folds CV score is {score}')\n    # Create a dataframe to store out of folds predictions\n    oof_df = pd.DataFrame({'customer_ID': train['customer_ID'], 'target': train[CFG.target], 'prediction': oof_predictions})\n    oof_df.to_csv(f'/content/drive/MyDrive/Amex/OOF/oof_lgbm_{CFG.boosting_type}_baseline_{CFG.n_folds}fold_seed{CFG.seed}.csv', index = False)\n    # Create a dataframe to store test prediction\n    test_df = pd.DataFrame({'customer_ID': test['customer_ID'], 'prediction': test_predictions})\n    test_df.to_csv(f'/content/drive/MyDrive/Amex/Predictions/test_lgbm_{CFG.boosting_type}_baseline_{CFG.n_folds}fold_seed{CFG.seed}.csv', index = False)\n    \n# seed_everything(CFG.seed)\n# train, test = read_data()\n# train_and_evaluate(train, test)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T13:59:54.323481Z","iopub.execute_input":"2022-07-14T13:59:54.323789Z","iopub.status.idle":"2022-07-14T13:59:54.356638Z","shell.execute_reply.started":"2022-07-14T13:59:54.323763Z","shell.execute_reply":"2022-07-14T13:59:54.355616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read Submission File\nThis is the submission file corresponding to the output of the previous pipeline (using the average blend of 3 seeds)","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('../input/amexsub797/sub.csv')\nsub.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T13:59:54.360552Z","iopub.execute_input":"2022-07-14T13:59:54.360889Z","iopub.status.idle":"2022-07-14T13:59:59.458593Z","shell.execute_reply.started":"2022-07-14T13:59:54.360860Z","shell.execute_reply":"2022-07-14T13:59:59.457599Z"},"trusted":true},"execution_count":null,"outputs":[]}]}