{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Objective\n\nMuch thanks to @ragnar123 for his notebook, and @raddar for his integer dataset (which I further processed).\nIt has been modified to run successfully within the memory restrictions of a Kaggle kernel by only training one fold on each run, with the models saved for prediction in a separate notebook.","metadata":{}},{"cell_type":"markdown","source":"# Training & Inference","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport os\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\nimport random\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nimport joblib\nimport itertools\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\nfrom itertools import combinations\n\n# ====================================================\n# Configurations\n# ====================================================\nclass CFG:\n    input_dir = '../input/amex-compress/'\n    seed = 39\n    n_folds = 5\n    target = 'target'\n\n# ====================================================\n# Seed everything\n# ====================================================\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \n\n\n# ====================================================\n# Read data\n# ====================================================\ndef read_data():\n    train = pd.read_parquet(CFG.input_dir + 'train_fe_compress.parquet')\n    train_section = np.array_split(train,5)[0]\n    del train\n    gc.collect()\n    test = pd.read_parquet(CFG.input_dir + 'test_fe_compress.parquet')\n    return train_section, test\n\n# ====================================================\n# Amex metric\n# ====================================================\ndef amex_metric(y_true, y_pred):\n    labels = np.transpose(np.array([y_true, y_pred]))\n    labels = labels[labels[:, 1].argsort()[::-1]]\n    weights = np.where(labels[:,0]==0, 20, 1)\n    cut_vals = labels[np.cumsum(weights) <= int(0.04 * np.sum(weights))]\n    top_four = np.sum(cut_vals[:,0]) / np.sum(labels[:,0])\n    gini = [0,0]\n    for i in [1,0]:\n        labels = np.transpose(np.array([y_true, y_pred]))\n        labels = labels[labels[:, i].argsort()[::-1]]\n        weight = np.where(labels[:,0]==0, 20, 1)\n        weight_random = np.cumsum(weight / np.sum(weight))\n        total_pos = np.sum(labels[:, 0] *  weight)\n        cum_pos_found = np.cumsum(labels[:, 0] * weight)\n        lorentz = cum_pos_found / total_pos\n        gini[i] = np.sum((lorentz - weight_random) * weight)\n    return 0.5 * (gini[1]/gini[0] + top_four)\n\n# ====================================================\n# LGBM amex metric\n# ====================================================\ndef lgb_amex_metric(y_pred, y_true):\n    y_true = y_true.get_label()\n    return 'amex_metric', amex_metric(y_true, y_pred), True","metadata":{"execution":{"iopub.status.busy":"2022-07-19T03:36:16.420655Z","iopub.execute_input":"2022-07-19T03:36:16.421392Z","iopub.status.idle":"2022-07-19T03:36:19.052592Z","shell.execute_reply.started":"2022-07-19T03:36:16.421289Z","shell.execute_reply":"2022-07-19T03:36:19.050440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(CFG.seed)\n#train, test = read_data()\n#train_and_evaluate(train, test)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T03:36:23.697198Z","iopub.execute_input":"2022-07-19T03:36:23.697702Z","iopub.status.idle":"2022-07-19T03:36:23.701969Z","shell.execute_reply.started":"2022-07-19T03:36:23.697665Z","shell.execute_reply":"2022-07-19T03:36:23.701330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Read train only\n# ====================================================\ndef read_train():\n    train = pd.read_parquet(CFG.input_dir + 'train_fe_compress.parquet')\n    return train\n# ====================================================\n#  Incremental Training\n# ====================================================\ndef train_model(train, x):\n    # Label encode categorical features\n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\"\n    ]\n    cat_features = [f\"{cf}_last\" for cf in cat_features]\n    \n    num_cols = list(train.dtypes[(train.dtypes == 'float32') | (train.dtypes == 'float64')].index)\n    num_cols = [col for col in num_cols if 'last' in col]\n    for col in num_cols:\n        train[col + '_round2'] = train[col].round(2)\n    num_cols = [col for col in train.columns if 'last' in col]\n    num_cols = [col[:-5] for col in num_cols if 'round' not in col]\n    for col in num_cols:\n        try:\n            train[f'{col}_last_mean_diff'] = train[f'{col}_last'] - train[f'{col}_mean']\n        except:\n            pass\n        \n    # Transform float64 and float32 to float16\n    num_cols = list(train.dtypes[(train.dtypes == 'float32') | (train.dtypes == 'float64')].index)\n    for col in tqdm(num_cols):\n        train[col] = train[col].astype(np.float16)\n    # Get feature list\n    features = [col for col in train.columns if col not in ['customer_ID', CFG.target]]\n    params = {\n        'objective': 'binary',\n        'metric': \"binary_logloss\",\n        'boosting': 'dart',\n        'seed': CFG.seed,\n        'num_leaves': 100,\n        'learning_rate': 0.01,\n        'feature_fraction': 0.20,\n        'bagging_freq': 10,\n        'bagging_fraction': 0.50,\n        'n_jobs': -1,\n        'lambda_l2': 2,\n        'min_data_in_leaf': 40,\n        'force_col_wise': True\n        }\n    #oof_predictions = np.zeros(len(train))\n    kfold = StratifiedKFold(n_splits = CFG.n_folds, shuffle = True, random_state = CFG.seed)\n    for fold, (trn_ind, val_ind) in enumerate(kfold.split(train, train[CFG.target])):\n        if fold == x:\n            print(' ')\n            print('-'*50)\n            print(f'Training fold {fold} with {len(features)} features...')\n\n            x_train, x_val = train[features].iloc[trn_ind], train[features].iloc[val_ind]\n            y_train, y_val = train[CFG.target].iloc[trn_ind], train[CFG.target].iloc[val_ind]\n\n            lgb_train = lgb.Dataset(x_train, y_train, categorical_feature = cat_features)\n            lgb_valid = lgb.Dataset(x_val, y_val, categorical_feature = cat_features)\n\n            model = lgb.train(\n                params = params,\n                train_set = lgb_train,\n                num_boost_round = 10500,\n                valid_sets = [lgb_train, lgb_valid],\n                early_stopping_rounds = 100,\n                verbose_eval = 500,\n                feval = lgb_amex_metric\n                )\n            # Save best model\n            joblib.dump(model, f'./lgbm_fold{fold}_seed{CFG.seed}.pkl')\n            # Predict validation\n            val_pred = model.predict(x_val)\n            # Add to out of folds array\n            #oof_predictions[val_ind] = val_pred\n            score = amex_metric(y_val, val_pred)\n            print(f'Our fold {fold} CV score is {score}')\n            del x_train, x_val, y_train, y_val, lgb_train, lgb_valid\n            gc.collect()\n    \n# ====================================================\n#  Helper\n# ====================================================\n\ndef helper(x):\n    train = read_train()\n    train_model(train, x)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T03:50:56.279920Z","iopub.execute_input":"2022-07-19T03:50:56.280372Z","iopub.status.idle":"2022-07-19T03:50:56.304196Z","shell.execute_reply.started":"2022-07-19T03:50:56.280339Z","shell.execute_reply":"2022-07-19T03:50:56.303262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#run n times for n folds, with parameter corresponding to the fold\nhelper(4)","metadata":{"execution":{"iopub.status.busy":"2022-07-19T03:50:56.629692Z","iopub.execute_input":"2022-07-19T03:50:56.630121Z","iopub.status.idle":"2022-07-19T04:03:24.086500Z","shell.execute_reply.started":"2022-07-19T03:50:56.630088Z","shell.execute_reply":"2022-07-19T04:03:24.084190Z"},"trusted":true},"execution_count":null,"outputs":[]}]}