{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Trying out PCA + Logistic Regression","metadata":{}},{"cell_type":"code","source":"!pip install feature-engine","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-10T15:43:44.899889Z","iopub.execute_input":"2022-08-10T15:43:44.900614Z","iopub.status.idle":"2022-08-10T15:43:55.841359Z","shell.execute_reply.started":"2022-08-10T15:43:44.900573Z","shell.execute_reply":"2022-08-10T15:43:55.840169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom colorama import Fore, Back, Style\nfrom sklearn.preprocessing import StandardScaler\nimport itertools\nfrom collections import defaultdict\nfrom sklearn.impute import KNNImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import RepeatedStratifiedKFold\nfrom sklearn.decomposition import PCA\nfrom sklearn.linear_model import LogisticRegression, HuberRegressor\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom feature_engine.encoding import WoEEncoder\nfrom scipy.stats import spearmanr, rankdata\n\nwarnings.filterwarnings('ignore')\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-10T15:43:55.844130Z","iopub.execute_input":"2022-08-10T15:43:55.844502Z","iopub.status.idle":"2022-08-10T15:43:55.854613Z","shell.execute_reply.started":"2022-08-10T15:43:55.844457Z","shell.execute_reply":"2022-08-10T15:43:55.853518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsubmission = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')\nprint(f'train {train.shape}, test {test.shape}')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:43:55.855997Z","iopub.execute_input":"2022-08-10T15:43:55.856316Z","iopub.status.idle":"2022-08-10T15:43:56.039803Z","shell.execute_reply.started":"2022-08-10T15:43:55.856286Z","shell.execute_reply":"2022-08-10T15:43:56.038553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.concat([train, test])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:43:56.042449Z","iopub.execute_input":"2022-08-10T15:43:56.042848Z","iopub.status.idle":"2022-08-10T15:43:56.062728Z","shell.execute_reply.started":"2022-08-10T15:43:56.042813Z","shell.execute_reply":"2022-08-10T15:43:56.061497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Inspired by the correlation matrix and missing indicator in @AMBROSM notebook: [tpsaug22-eda-which-makes-sense](https://www.kaggle.com/code/ambrosm/tpsaug22-eda-which-makes-sense)\n- After creating **m3_missing** and **m5_missing**, I use **HuberRegressor** to fill different product_code's measurement_17 separately.","metadata":{}},{"cell_type":"code","source":"_, axs = plt.subplots(3, 3, figsize=(18, 18))\nfor product, ax in zip(np.unique(data.product_code), axs.ravel()):\n    corr = data.loc[data.product_code == product, [f'measurement_{i}' for i in range(3, 18)]].corr()\n    mask = np.triu(np.ones_like(corr, dtype=bool))\n    sns.heatmap(corr*10, mask=mask, linewidth=0.0, fmt='.0f', \n                annot=True, annot_kws={'size': 8}, \n                cmap='twilight_shifted_r', center=0, ax=ax, cbar=False)\n    ax.set_title(f'product code: {product}')\nplt.tight_layout(w_pad=0.5)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:43:56.064315Z","iopub.execute_input":"2022-08-10T15:43:56.064792Z","iopub.status.idle":"2022-08-10T15:44:07.945642Z","shell.execute_reply.started":"2022-08-10T15:43:56.064746Z","shell.execute_reply":"2022-08-10T15:44:07.944316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['m3_missing'] = data['measurement_3'].isnull().astype(np.int8)\ndata['m5_missing'] = data['measurement_5'].isnull().astype(np.int8)\ndata['area'] = data['attribute_2'] * data['attribute_3']\n\nfeature = [f for f in test.columns if f.startswith('measurement') or f=='loading']\n\n# use highly correlative features to train HuberRegressor model. \nfill_dict = {\n    'A': ['measurement_5','measurement_6','measurement_8'],\n    'B': ['measurement_4','measurement_5','measurement_7'],\n    'C': ['measurement_5','measurement_7','measurement_8','measurement_9'],\n    'D': ['measurement_5','measurement_6','measurement_7','measurement_8'],\n    'E': ['measurement_4','measurement_5','measurement_6','measurement_8'],\n    'F': ['measurement_4','measurement_5','measurement_6','measurement_7'],\n    'G': ['measurement_4','measurement_6','measurement_8','measurement_9'],\n    'H': ['measurement_4','measurement_5','measurement_7','measurement_8','measurement_9'],\n    'I': ['measurement_3','measurement_7','measurement_8']\n}\n\nfeature = [f for f in data.columns if f.startswith('measurement') or f=='loading']\n\nfor code in data.product_code.unique():\n    tmp = data[data.product_code==code]\n    column = fill_dict[code]\n    tmp_train = tmp[column+['measurement_17']].dropna(how='any')\n    tmp_test = tmp[(tmp[column].isnull().sum(axis=1)==0)&(tmp['measurement_17'].isnull())]\n    model = HuberRegressor(epsilon=1.9)\n    model.fit(tmp_train[column], tmp_train['measurement_17'])\n    data.loc[(data.product_code==code)&(data[column].isnull().sum(axis=1)==0)&(data['measurement_17'].isnull()), 'measurement_17'] = model.predict(tmp_test[column])\n\n    model2 = KNNImputer(n_neighbors=3)\n    data.loc[data.product_code==code, feature] = model2.fit_transform(data.loc[data.product_code==code, feature])\n    print(f\"code {code} has been filled {len(tmp_test)} samples, KNN imputing finished\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:44:07.947598Z","iopub.execute_input":"2022-08-10T15:44:07.948097Z","iopub.status.idle":"2022-08-10T15:44:19.479735Z","shell.execute_reply.started":"2022-08-10T15:44:07.948048Z","shell.execute_reply":"2022-08-10T15:44:19.478536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"code_r = defaultdict(int)\nfor code in data.product_code.unique():\n    for (i, j) in itertools.combinations([f'measurement_{n}' for n in range(0, 18)], 2):\n        r, p = spearmanr(data[data.product_code==code][i], data[data.product_code==code][j])\n        if p < 0.05 and (i, j) in (('measurement_0', 'measurement_1'), ('measurement_0', 'measurement_2'), ('measurement_1', 'measurement_2')):\n            if r > 0.2:\n                code_r[(i, j)] += 1\n        elif p < 0.05:\n            if r > 0.2:\n                code_r[(i, j)] += 1","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:44:19.481047Z","iopub.execute_input":"2022-08-10T15:44:19.481388Z","iopub.status.idle":"2022-08-10T15:44:34.883056Z","shell.execute_reply.started":"2022-08-10T15:44:19.481355Z","shell.execute_reply":"2022-08-10T15:44:34.881686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"code_r","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:44:34.884862Z","iopub.execute_input":"2022-08-10T15:44:34.885231Z","iopub.status.idle":"2022-08-10T15:44:34.894124Z","shell.execute_reply.started":"2022-08-10T15:44:34.885198Z","shell.execute_reply":"2022-08-10T15:44:34.892802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- **??? How to use measurement_0 ~ measurement_16 ???**","metadata":{}},{"cell_type":"code","source":"data['measurement_avg'] = data[[f'measurement_{i}' for i in range(3, 17)]].mean(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:44:34.895860Z","iopub.execute_input":"2022-08-10T15:44:34.896357Z","iopub.status.idle":"2022-08-10T15:44:34.910016Z","shell.execute_reply.started":"2022-08-10T15:44:34.896310Z","shell.execute_reply":"2022-08-10T15:44:34.908734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _scale(train_data, val_data, test_data, feats):\n    scaler = StandardScaler()\n    # scaler = PowerTransformer()\n    \n    scaled_train = scaler.fit_transform(train_data[feats])\n    scaled_val = scaler.transform(val_data[feats])\n    scaled_test = scaler.transform(test_data[feats])\n    \n    #back to dataframe\n    new_train = train_data.copy()\n    new_val = val_data.copy()\n    new_test = test_data.copy()\n    \n    new_train[feats] = scaled_train\n    new_val[feats] = scaled_val\n    new_test[feats] = scaled_test\n    \n    assert len(train_data) == len(new_train)\n    assert len(val_data) == len(new_val)\n    assert len(test_data) == len(new_test)\n    \n    return new_train, new_val, new_test","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:44:34.913929Z","iopub.execute_input":"2022-08-10T15:44:34.914315Z","iopub.status.idle":"2022-08-10T15:44:34.924220Z","shell.execute_reply.started":"2022-08-10T15:44:34.914281Z","shell.execute_reply":"2022-08-10T15:44:34.922817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = data[data.failure.notnull()]\ntest = data[data.failure.isnull()]\nprint(train.shape, test.shape)\n\ngroups = train.product_code\nX = train.drop(['failure'], axis=1)\ny = train['failure'].astype(int)\ntest = test.drop(['failure'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:44:34.926690Z","iopub.execute_input":"2022-08-10T15:44:34.927307Z","iopub.status.idle":"2022-08-10T15:44:34.956387Z","shell.execute_reply.started":"2022-08-10T15:44:34.927246Z","shell.execute_reply":"2022-08-10T15:44:34.954985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Thanks to @MAXSARMENTO [standing-on-the-shoulder-of-giants](https://www.kaggle.com/code/maxsarmento/lb-0-58978-standing-on-the-shoulder-of-giants?scriptVersionId=102785631)\n- Use **Weight of Evidence** to encode attribute_0","metadata":{}},{"cell_type":"code","source":"woe_encoder = WoEEncoder(variables=['attribute_0'])\nwoe_encoder.fit(X, y)\nX = woe_encoder.transform(X)\ntest = woe_encoder.transform(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:44:34.958103Z","iopub.execute_input":"2022-08-10T15:44:34.958509Z","iopub.status.idle":"2022-08-10T15:44:35.018923Z","shell.execute_reply.started":"2022-08-10T15:44:34.958473Z","shell.execute_reply":"2022-08-10T15:44:35.017824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"select_feature = ['loading', 'attribute_0', 'measurement_17', 'measurement_0', 'measurement_1',\n                  'measurement_2', 'area', 'm3_missing', 'm5_missing', 'measurement_avg']","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:44:35.020586Z","iopub.execute_input":"2022-08-10T15:44:35.021202Z","iopub.status.idle":"2022-08-10T15:44:35.026975Z","shell.execute_reply.started":"2022-08-10T15:44:35.021156Z","shell.execute_reply":"2022-08-10T15:44:35.025999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"def get_model():\n    model = Pipeline([('pca', PCA(n_components=5)), ('l', LogisticRegression(max_iter=200, C=0.0001, penalty='l2', solver='newton-cg'))])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:44:35.028532Z","iopub.execute_input":"2022-08-10T15:44:35.028950Z","iopub.status.idle":"2022-08-10T15:44:35.038985Z","shell.execute_reply.started":"2022-08-10T15:44:35.028910Z","shell.execute_reply":"2022-08-10T15:44:35.037629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\nimportance_list = []\n\n#kf = GroupKFold(n_splits=5)\nkf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)\nfor i, (train_idx, val_idx) in enumerate(kf.split(X, y)):\n    x_train, x_val = X.iloc[train_idx], X.iloc[val_idx]\n    y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n    x_train, x_val, x_test = _scale(x_train, x_val, test, select_feature)\n    \n    model = get_model()\n    model.fit(x_train[select_feature], y_train)\n\n    val_preds = model.predict_proba(x_val[select_feature])[:, 1]\n    auc_score = roc_auc_score(y_val, val_preds)\n    models[auc_score] = model\n    print(\"FOLD: \", i+1, \" ROC-AUC:\", auc_score)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:51:54.190578Z","iopub.execute_input":"2022-08-10T15:51:54.191049Z","iopub.status.idle":"2022-08-10T15:51:55.483721Z","shell.execute_reply.started":"2022-08-10T15:51:54.191015Z","shell.execute_reply":"2022-08-10T15:51:55.482089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Select top 3 models","metadata":{}},{"cell_type":"code","source":"preds = np.zeros((3, len(x_test)))\nfor i, m in enumerate(list(models.keys())[:2]):\n    preds[i] = models[m].predict_proba(x_test[select_feature])[:, 1]","metadata":{"execution":{"iopub.status.busy":"2022-08-10T15:54:06.975125Z","iopub.execute_input":"2022-08-10T15:54:06.975552Z","iopub.status.idle":"2022-08-10T15:54:07.006220Z","shell.execute_reply.started":"2022-08-10T15:54:06.975516Z","shell.execute_reply":"2022-08-10T15:54:07.004557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"submission['failure'] = np.mean(preds, axis=0)\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:25:13.397474Z","iopub.execute_input":"2022-08-10T03:25:13.398603Z","iopub.status.idle":"2022-08-10T03:25:13.458474Z","shell.execute_reply.started":"2022-08-10T03:25:13.398561Z","shell.execute_reply":"2022-08-10T03:25:13.457206Z"},"trusted":true},"execution_count":null,"outputs":[]}]}