{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Standing on the shoulders of giants that are on the shoulders of other giants\n\nThe basis of this notebook is the one created by @maxsarmento (https://www.kaggle.com/code/maxsarmento/lb-0-58978-standing-on-the-shoulder-of-giants/notebook) which in turn is basically the one suggested by @desalegngeb  (https://www.kaggle.com/code/desalegngeb/tps08-logisticregression-and-some-fe/notebook?scriptVersionId=102691691) so credits for them.","metadata":{}},{"cell_type":"markdown","source":"### Purpose\nMainly, I created this notebook to play around with Feature Engineering. After several trials I found 3 potential features:\n\n- One hot encoding attribute_2*3\n- Join attribute_0 with attribute_2*3 and one hot encoding\n- Groupby attribute_2*3 and take the mean of loading\n\nHere is the associated post where I share the insights (https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/342938).\n\nThe final version of the notebook is the one with the highest LB score, which keeps the original features (because new features improve CV but hurt LB) and make the final submission via folds bagging.","metadata":{}},{"cell_type":"code","source":"# IMPORTS\nimport os\nimport sys\nimport joblib\n\nimport numpy as np \nimport pandas as pd\n\nfrom sklearn.preprocessing import KBinsDiscretizer\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import roc_auc_score, log_loss\n\n!pip install feature_engine\nfrom feature_engine.encoding import WoEEncoder\n\n!git clone https://github.com/analokmaus/kuma_utils.git\nsys.path.append(\"kuma_utils/\")\nfrom kuma_utils.preprocessing.imputer import LGBMImputer\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport gc\ngc.enable()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-09T10:25:28.024838Z","iopub.execute_input":"2022-08-09T10:25:28.025683Z","iopub.status.idle":"2022-08-09T10:25:40.948673Z","shell.execute_reply.started":"2022-08-09T10:25:28.025640Z","shell.execute_reply":"2022-08-09T10:25:40.946927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CREATE FOLDS\ndef create_folds(df_train):\n    cv = StratifiedKFold(n_splits=CFG.N_SPLITS, shuffle=True, random_state=CFG.SEED)\n#     cv = GroupKFold()\n    df_train[\"fold\"] = -1\n    for fold, (_, val_idx) in enumerate(cv.split(df_train, df_train[\"failure\"])):\n        df_train.loc[val_idx, \"fold\"] = fold\n    return df_train\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T10:25:40.952503Z","iopub.execute_input":"2022-08-09T10:25:40.953606Z","iopub.status.idle":"2022-08-09T10:25:40.963005Z","shell.execute_reply.started":"2022-08-09T10:25:40.953531Z","shell.execute_reply":"2022-08-09T10:25:40.961501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CROSSVALIDATION\ndef crossvalidation(df_train, df_test, features, params):\n    \n    score_train_list = []\n    score_val_list = []\n    \n    oofs = df_train[[\"failure\", \"fold\"]].copy()\n    oofs[\"preds\"] = None\n\n    for fold in range(CFG.N_SPLITS):\n\n        train_idx = df_train[\"fold\"]!=fold\n        val_idx = df_train[\"fold\"]==fold\n        \n        X_train_cv = df_train.loc[train_idx][features]\n        X_val_cv = df_train.loc[val_idx][features]\n        \n        y_train_cv = df_train.loc[train_idx][\"failure\"]\n        y_val_cv = df_train.loc[val_idx][\"failure\"]\n        \n        model = LogisticRegression(**params)\n        model.fit(X_train_cv, y_train_cv)\n        \n        joblib.dump(model, os.path.join(CFG.OUT_PATH, f\"model_LR_fold{fold}.pkl\"))\n        \n        y_train_cv_pred = model.predict_proba(X_train_cv)[:,1]\n        y_val_cv_pred = model.predict_proba(X_val_cv)[:,1]\n        \n        oofs.loc[val_idx, \"preds\"] = y_val_cv_pred\n\n        score_train = roc_auc_score(y_train_cv, y_train_cv_pred)\n        score_val = roc_auc_score(y_val_cv, y_val_cv_pred)\n        \n        print(f\"Train score: {score_train:.7f} / val score: {score_val:.7f}\")\n        \n        score_train_list.append(score_train)\n        score_val_list.append(score_val)\n    \n    oofs.to_csv(os.path.join(CFG.OUT_PATH, f\"oofs_seed{CFG.SEED}.csv\"))\n    \n    score_train_avg = np.mean(score_train_list)\n    score_val_avg = np.mean(score_val_list)\n    score_val_ove = roc_auc_score(oofs[\"failure\"], oofs[\"preds\"])\n        \n    print(f\"Train avg score: {score_train_avg:.7f} / val avg score: {score_val_avg:.7f}\")\n    print(f\"OOFs overall score: {score_val_ove:.7f}\")\n\n\n    \n# INFERENCE\ndef inference(df_train, df_test, features, params):\n    \n    X_train = df_train[features]\n    y_train = df_train[\"failure\"]\n    X_test = df_test[features]\n    y_test_pred = np.zeros(len(X_test))\n    \n    if CFG.INFER_MODE == \"fold\":\n        for fold in range(CFG.N_SPLITS):\n            model = joblib.load(os.path.join(CFG.OUT_PATH, f\"model_LR_fold{fold}.pkl\"))\n            y_test_pred += model.predict_proba(X_test)[:,1] / CFG.N_SPLITS\n    \n    elif CFG.INFER_MODE == \"all\":\n        model = LogisticRegression(**params)\n        model.fit(X_train, y_train)\n        y_test_pred = model.predict_proba(X_test)[:,1]\n            \n    sub[\"failure\"] = y_test_pred\n    sub.to_csv(os.path.join(CFG.OUT_PATH, f\"submission_seed{CFG.SEED}.csv\"), index=False)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-09T10:26:23.271592Z","iopub.execute_input":"2022-08-09T10:26:23.272016Z","iopub.status.idle":"2022-08-09T10:26:23.290852Z","shell.execute_reply.started":"2022-08-09T10:26:23.271982Z","shell.execute_reply":"2022-08-09T10:26:23.289620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PREPROCESSING\ndef preprocessing(df_train, df_test):\n     \n    # Missing indicator\n    for df in [df_train, df_test]:\n        df[\"m_3_missing\"] = df[\"measurement_3\"].isnull().astype(int)\n        df[\"m_5_missing\"] = df[\"measurement_5\"].isnull().astype(int)\n\n        \n    # Missing value imputation\n    imptr = LGBMImputer(n_iter=50)\n    def impute_nans(df_train, df_test):\n        nan_features = [col for col in df_train.columns if df_train[col].isnull().any()]\n        for pc in df_train[\"product_code\"].unique():\n            df_train.loc[df_train[\"product_code\"]==pc, nan_features] = imptr.fit_transform(df_train.loc[df_train[\"product_code\"]==pc, nan_features])\n        \n        nan_features = [col for col in df_test.columns if df_test[col].isnull().any()]\n        for pc in df_test[\"product_code\"].unique():\n            df_test.loc[df_test[\"product_code\"]==pc, nan_features] = imptr.fit_transform(df_test.loc[df_test[\"product_code\"]==pc, nan_features])\n\n        return df_train, df_test\n    \n    df_train, df_test = impute_nans(df_train, df_test)\n    \n    \n    # Area\n    for df in [df_train, df_test]:\n        df[\"attribute_2*3\"] = df[\"attribute_2\"] * df[\"attribute_3\"]\n    \n    \n    # Aggregations\n    meas_gr1_cols = [f\"measurement_{i:d}\" for i in list(range(3, 5)) + list(range(9, 17))]\n    meas_gr2_cols = [f\"measurement_{i:d}\" for i in list(range(5, 9))]\n    for df in [df_train, df_test]:\n        df[\"meas_gr1_avg\"] = np.mean(df[meas_gr1_cols], axis=1)\n        df[\"meas_gr1_std\"] = np.std(df[meas_gr1_cols], axis=1)\n        df[\"meas_gr2_avg\"] = np.mean(df[meas_gr2_cols], axis=1)\n    \n    for df in [df_train, df_test]:\n        df[\"meas17/meas_gr2_avg\"] = df[\"measurement_17\"] / df[\"meas_gr2_avg\"]\n    \n    \n#   # New features\n#     df_all = pd.concat([df_train, df_test], axis=0)\n#     df_all[\"gb_attribute_2*3_mean_loading\"] = df_all[\"attribute_2*3\"].map(df_all.groupby(\"attribute_2*3\")[\"loading\"].mean())\n    \n#     ohe_att_23 = pd.get_dummies(df_all[\"attribute_2*3\"], prefix=\"attribute_2*3\")\n#     df_all[ohe_att_23.columns] = ohe_att_23\n    \n#     df_all[\"attribute_0_attribute_2*3\"] = df_all[\"attribute_0\"] + \"_\" + df_all[\"attribute_2*3\"].astype(str)\n#     ohe_att_0_att_23 = pd.get_dummies(df_all[\"attribute_0_attribute_2*3\"], prefix=\"attribute_0_attribute_2*3\")\n#     df_all[ohe_att_0_att_23.columns] = ohe_att_0_att_23\n    \n#     df_train = df_all.iloc[0:len(df_train)]\n#     df_test = df_all.iloc[len(df_train):]\n    \n    \n    # WoE encoding\n    woe_encoder = WoEEncoder(variables=[\"attribute_0\"])\n    df_train[\"attribute_0\"] = woe_encoder.fit_transform(df_train[\"attribute_0\"].to_frame(), df_train[\"failure\"])\n    df_test[\"attribute_0\"] = woe_encoder.transform(df_test[\"attribute_0\"].to_frame())\n    \n    \n    features = [\"attribute_0\", \"measurement_0\", \"measurement_1\", \"measurement_2\", \"m_3_missing\", \"m_5_missing\",\n               \"meas_gr1_avg\", \"meas_gr1_std\", \"attribute_2*3\", \"loading\", \"measurement_17\", \"meas17/meas_gr2_avg\"] \\\n#                 + list(ohe_att_0_att_23.columns) + [\"gb_attribute_2*3_mean_loading\"] + list(ohe_att_23.columns)\n    \n#     del df_all, ohe_att_23, ohe_att_0_att_23\n#     gc.collect()\n    \n    return df_train, df_test, features","metadata":{"execution":{"iopub.status.busy":"2022-08-09T10:26:24.807609Z","iopub.execute_input":"2022-08-09T10:26:24.808079Z","iopub.status.idle":"2022-08-09T10:26:24.832247Z","shell.execute_reply.started":"2022-08-09T10:26:24.808042Z","shell.execute_reply":"2022-08-09T10:26:24.831321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    INPUT_PATH = \"/kaggle/input/tabular-playground-series-aug-2022/\"\n    OUT_PATH = \"/kaggle/working/\"\n    N_SPLITS = 5\n    SEED = 0\n    INFER_MODE = \"fold\"\n\nparams_LR = {\n    \"max_iter\": 500, \"C\": 0.0001, \"penalty\": \"l2\", \"solver\": \"newton-cg\"\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-09T10:26:26.106562Z","iopub.execute_input":"2022-08-09T10:26:26.106967Z","iopub.status.idle":"2022-08-09T10:26:26.114934Z","shell.execute_reply.started":"2022-08-09T10:26:26.106933Z","shell.execute_reply":"2022-08-09T10:26:26.113150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(CFG.INPUT_PATH, \"train.csv\"))\ndf_test = pd.read_csv(os.path.join(CFG.INPUT_PATH, \"test.csv\"))\nsub = pd.read_csv(os.path.join(CFG.INPUT_PATH, \"sample_submission.csv\"))","metadata":{"execution":{"iopub.status.busy":"2022-08-09T10:26:27.450907Z","iopub.execute_input":"2022-08-09T10:26:27.451810Z","iopub.status.idle":"2022-08-09T10:26:27.703029Z","shell.execute_reply.started":"2022-08-09T10:26:27.451768Z","shell.execute_reply":"2022-08-09T10:26:27.701690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = create_folds(df_train)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T10:26:28.786852Z","iopub.execute_input":"2022-08-09T10:26:28.787342Z","iopub.status.idle":"2022-08-09T10:26:28.818671Z","shell.execute_reply.started":"2022-08-09T10:26:28.787302Z","shell.execute_reply":"2022-08-09T10:26:28.817680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_test, features = preprocessing(df_train, df_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T10:26:30.126626Z","iopub.execute_input":"2022-08-09T10:26:30.127622Z","iopub.status.idle":"2022-08-09T10:26:53.911563Z","shell.execute_reply.started":"2022-08-09T10:26:30.127579Z","shell.execute_reply":"2022-08-09T10:26:53.910380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crossvalidation(df_train, df_test, features, params_LR)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T10:26:53.914208Z","iopub.execute_input":"2022-08-09T10:26:53.915433Z","iopub.status.idle":"2022-08-09T10:26:57.402784Z","shell.execute_reply.started":"2022-08-09T10:26:53.915381Z","shell.execute_reply":"2022-08-09T10:26:57.401626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inference(df_train, df_test, features, params_LR)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T09:05:54.728413Z","iopub.status.idle":"2022-08-09T09:05:54.728852Z","shell.execute_reply.started":"2022-08-09T09:05:54.728644Z","shell.execute_reply":"2022-08-09T09:05:54.728671Z"},"trusted":true},"execution_count":null,"outputs":[]}]}