{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Summary:\nThis notebook provides a reusable ```ProductSplitter3v2``` cross validator, which implements strategy 3 from [GroupKFold isn't enough](https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/341896).\n\nEverything after the class definition is just present to demonstrate how you might use it.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.impute import KNNImputer\nfrom dataclasses import dataclass\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.linear_model import RidgeClassifier\nfrom sklearn.model_selection import cross_validate, KFold","metadata":{"execution":{"iopub.status.busy":"2022-08-05T19:19:18.088736Z","iopub.execute_input":"2022-08-05T19:19:18.089611Z","iopub.status.idle":"2022-08-05T19:19:19.545577Z","shell.execute_reply.started":"2022-08-05T19:19:18.089483Z","shell.execute_reply":"2022-08-05T19:19:19.544141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The actual ```ProductSplitter3v2``` class, ready to use in your notebooks.","metadata":{}},{"cell_type":"code","source":"@dataclass\nclass ProductSplitter3v2():\n    \"\"\"A speclialized CrossValidator for TPS2208 which splits between product codes. Each split includes 3\n    products in the train set and 2 in the validation set.\"\"\"\n    def split(self, X: pd.DataFrame, _y=None, _groups=None):\n        indices = list(X.groupby(\"product_code\").indices.values())\n        for i in range(len(indices)):\n            for j in range(i + 1, len(indices)):\n                yield [\n                    np.concatenate([ix for k, ix in enumerate(indices) if k not in [i, j]]),\n                    np.concatenate([ix for k, ix in enumerate(indices) if k in [i, j]])\n                ]\n","metadata":{"execution":{"iopub.status.busy":"2022-08-05T19:19:19.548122Z","iopub.execute_input":"2022-08-05T19:19:19.548591Z","iopub.status.idle":"2022-08-05T19:19:19.559243Z","shell.execute_reply.started":"2022-08-05T19:19:19.548534Z","shell.execute_reply":"2022-08-05T19:19:19.557820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Leak-free imputation, as described in [Per-product-code Imputation](https://www.kaggle.com/code/purist1024/per-product-code-imputation)","metadata":{}},{"cell_type":"code","source":"def kimpute(X: pd.DataFrame, n=10, weights=\"uniform\"):\n    \"\"\"Impute missing values in TPS2208 data.\n    \n    Imputation is performed over separate \"per-product-code\" batches, and is designed to leave all non-imputed \n    data in the exact same format as before imputation.\"\"\"\n    def transform(X):\n        return pd.DataFrame(\n            KNNImputer(n_neighbors=n, weights=weights).fit_transform(X), index=X.index,\n            columns=X.columns)\n\n    cats = [\"product_code\", \"attribute_0\", \"attribute_1\", \"attribute_2\", \"attribute_3\"]\n    ints = [\"measurement_0\", \"measurement_1\", \"measurement_2\"]\n    right = pd.concat([transform(gdf.drop(columns=cats)) for g, gdf in X.groupby(\"product_code\")],\n                      axis=\"rows\")\n    right[ints] = right[ints].round().astype(int)\n    return pd.concat([X[cats], right], axis=\"columns\").reindex(columns=X.columns)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-05T19:19:19.561461Z","iopub.execute_input":"2022-08-05T19:19:19.562314Z","iopub.status.idle":"2022-08-05T19:19:19.576283Z","shell.execute_reply.started":"2022-08-05T19:19:19.562270Z","shell.execute_reply":"2022-08-05T19:19:19.575437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load and impute data","metadata":{}},{"cell_type":"code","source":"Xy_train = pd.read_csv(\"../input/tabular-playground-series-aug-2022/train.csv\", index_col='id')\nXt, yt = kimpute(Xy_train.drop(columns=[\"failure\"])), Xy_train.failure\n","metadata":{"execution":{"iopub.status.busy":"2022-08-05T19:19:19.579287Z","iopub.execute_input":"2022-08-05T19:19:19.580097Z","iopub.status.idle":"2022-08-05T19:19:27.190297Z","shell.execute_reply.started":"2022-08-05T19:19:19.580052Z","shell.execute_reply":"2022-08-05T19:19:27.189105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A trivial transformer which deletes everything but the *loading* column. As noted elsewhere, this one feature is competitive with a lot of basic feature engineering strategies.","metadata":{}},{"cell_type":"code","source":"@dataclass\nclass TransformTPS2208(BaseEstimator, TransformerMixin):\n    def fit(self, X, y=None):\n        return self # We don't need to train this transformer\n    def transform(self, X):\n        return X[['loading']]\n","metadata":{"execution":{"iopub.status.busy":"2022-08-05T19:19:27.191627Z","iopub.execute_input":"2022-08-05T19:19:27.191997Z","iopub.status.idle":"2022-08-05T19:19:27.199279Z","shell.execute_reply.started":"2022-08-05T19:19:27.191964Z","shell.execute_reply":"2022-08-05T19:19:27.198023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I like using Transformers and Pipelines, because it makes CV testing very clear and eliminates leaks.","metadata":{}},{"cell_type":"code","source":"model = make_pipeline(TransformTPS2208(), RidgeClassifier(random_state=1))\n","metadata":{"execution":{"iopub.status.busy":"2022-08-05T19:19:27.201038Z","iopub.execute_input":"2022-08-05T19:19:27.201592Z","iopub.status.idle":"2022-08-05T19:19:27.214877Z","shell.execute_reply.started":"2022-08-05T19:19:27.201560Z","shell.execute_reply":"2022-08-05T19:19:27.213609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The wrong way to cross-validate, with random sampling rather than grouping by product code.","metadata":{}},{"cell_type":"code","source":"cv = cross_validate(model, Xt, yt, cv=KFold(10, shuffle=True, random_state=0), \n               scoring='roc_auc')\nprint(\"Scores:\", cv['test_score'])\nprint(f\"avg={np.mean(cv['test_score'])}; std={np.std(cv['test_score'])}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-05T19:19:27.216505Z","iopub.execute_input":"2022-08-05T19:19:27.217078Z","iopub.status.idle":"2022-08-05T19:19:27.571861Z","shell.execute_reply.started":"2022-08-05T19:19:27.217044Z","shell.execute_reply":"2022-08-05T19:19:27.570358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Our new product-based splitter. It looks pretty similar, but it should be much more trustworthy.","metadata":{}},{"cell_type":"code","source":"cv = cross_validate(model, Xt, yt, cv=ProductSplitter3v2(), scoring='roc_auc')\nprint(\"Scores:\", cv['test_score'])\nprint(f\"avg={np.mean(cv['test_score'])}; std={np.std(cv['test_score'])}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-05T19:19:27.573861Z","iopub.execute_input":"2022-08-05T19:19:27.574724Z","iopub.status.idle":"2022-08-05T19:19:27.957927Z","shell.execute_reply.started":"2022-08-05T19:19:27.574661Z","shell.execute_reply":"2022-08-05T19:19:27.956407Z"},"trusted":true},"execution_count":null,"outputs":[]}]}